--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:25.865506Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b0164-855bf76f5c-9lgt7 to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:25Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c68476490c08e4 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: f822c5d1-b4fe-4b53-a8e0-84667ec7e466 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: "2026-07-28T17:32:26Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:26Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c68476629b9ba4 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 5d993946-21b0-47ac-9258-a651fd08b1b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b0164} kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: "2026-07-28T17:32:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:26Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c6847663e37446 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: af6eb4db-b281-461f-ba4b-b5b379c66756 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b0164} kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: "2026-07-28T17:32:26Z" message: 'Created container: ensemble-graph-b0164' metadata: creationTimestamp: "2026-07-28T17:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:26Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c684766714f4fc namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 1581f6ae-f829-4264-a747-0eb9be888682 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b0164} kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: "2026-07-28T17:32:26Z" message: Started container ensemble-graph-b0164 metadata: creationTimestamp: "2026-07-28T17:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:26Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c684766787f1b2 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: ecf1ff91-db03-4c24-9f17-aa4ea8367a60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b0164} kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: "2026-07-28T17:32:35Z" message: Stopping container ensemble-graph-b0164 metadata: creationTimestamp: "2026-07-28T17:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:35Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c68478a19b80ff namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 97dac114-dbb0-43d0-a423-38c213fe8233 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b0164} kind: Pod name: ensemble-graph-b0164-855bf76f5c-9lgt7 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: b9cf40c6-297f-4f91-b813-1f04685df5e0 kind: Event lastTimestamp: "2026-07-28T17:33:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:02Z" name: ensemble-graph-b0164-855bf76f5c-9lgt7.18c68478fae7ff17 namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 67c41107-e4b8-41e5-ac25-2f659aa84537 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b0164-855bf76f5c namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: c24a273c-b49f-4d30-97fe-8eab03c89ed8 kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: 'Created pod: ensemble-graph-b0164-855bf76f5c-9lgt7' metadata: creationTimestamp: "2026-07-28T17:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:25Z" name: ensemble-graph-b0164-855bf76f5c.18c68476486c6edb namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: e41ce9bd-759e-4263-83c6-d4b87e9dd397 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 6c681937-4ec4-4df4-a217-f23c1acb07e9 kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: Scaled up replica set ensemble-graph-b0164-855bf76f5c from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:25Z" name: ensemble-graph-b0164.18c6847647ad7278 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: b9dc7544-c699-4371-830a-a8fbdb1bb81a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: 599d9a92-6919-4116-9859-2be3badc78ba kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: InferenceGraph [ensemble-graph-b0164] is Ready metadata: creationTimestamp: "2026-07-28T17:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:25Z" name: ensemble-graph-b0164.18c684764d33166f namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 081fbb83-cf33-4642-a9da-cee49332aef9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:36.208858Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-fca3b-6cff5d8565-22kw6 to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:36Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c68486a9d66226 namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: 967ca6f2-956f-4001-a308-25699832fdf7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-fca3b-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:36Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c68486b10057a2 namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: fd129ad6-e49b-4c1a-a362-6e960912b283 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:37Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21691" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:33:37Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:37Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c68486e6f0c687 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 6982a5fb-0cba-4987-b8a9-5a1781c59d0f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fca3b} kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:33:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:37Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c68486e8354441 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: b4c32c09-2846-4d22-8f03-2a8d9f0df0fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fca3b} kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:33:37Z" message: 'Created container: ensemble-graph-fca3b' metadata: creationTimestamp: "2026-07-28T17:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:37Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c68486eb2fe92a namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 447cd735-85aa-40e3-85d8-00956f687aa6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fca3b} kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:33:37Z" message: Started container ensemble-graph-fca3b metadata: creationTimestamp: "2026-07-28T17:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:37Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c68486eba926e3 namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: dd97ad37-13a0-464c-98cc-d596bfef286c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fca3b} kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:41:50Z" message: Stopping container ensemble-graph-fca3b metadata: creationTimestamp: "2026-07-28T17:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:50Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c684f9d7a27209 namespace: kserve-ci-e2e-test resourceVersion: "25668" uid: cec19694-9789-45f4-985a-ea9f326e2581 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:41:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fca3b} kind: Pod name: ensemble-graph-fca3b-6cff5d8565-22kw6 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 78455cfc-f209-4387-b557-2cecb243fbad kind: Event lastTimestamp: "2026-07-28T17:42:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:18Z" name: ensemble-graph-fca3b-6cff5d8565-22kw6.18c684fa81e70097 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: d6ac5669-36e4-487d-96cd-29bdf90030ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-fca3b-6cff5d8565 namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 72e8db1d-138a-4d64-848f-d3c5d3ebaf47 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'Created pod: ensemble-graph-fca3b-6cff5d8565-22kw6' metadata: creationTimestamp: "2026-07-28T17:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: ensemble-graph-fca3b-6cff5d8565.18c68486a942c22d namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: b6fe7477-d425-4a2c-9489-116c8337de70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: b08ef1a3-0536-45f9-9769-186d2d8944e3 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: Scaled up replica set ensemble-graph-fca3b-6cff5d8565 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: ensemble-graph-fca3b.18c68486a88b44d6 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: df53cca1-30c8-4a00-b225-e08ec77e6da4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 06c3ca0c-1f55-4b30-882c-b7a735ec74cd kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: InferenceGraph [ensemble-graph-fca3b] is Ready metadata: creationTimestamp: "2026-07-28T17:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:36Z" name: ensemble-graph-fca3b.18c68486ad8ec0df namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 3d1166c9-e045-410d-9284-fd3947179099 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: 462640bf-297f-422a-9c6c-b9da9b6f40ae kind: Event lastTimestamp: "2026-07-28T17:34:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:06Z" name: ensemble-graph-fca3b.18c6848a27d86d58 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: cddf6ce6-4c0d-436e-b9a5-7da1f3f7222a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: 462640bf-297f-422a-9c6c-b9da9b6f40ae kind: Event lastTimestamp: "2026-07-28T17:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:06Z" name: ensemble-graph-fca3b.18c6848a2815dfb4 namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 90af177f-1dd5-4fbf-b443-5a4241ff5a50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:21.694868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c6847550752849 namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 52c73433-2f6c-49f2-8bec-aca9ca322068 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1bf99-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c684755b86c6c4 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: a9b8cef5-45cd-402d-a234-52f6c3880a9d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475c32a4370 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 55c9106b-ec4a-407e-b5bc-76cc8f0cf91d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475c4bda5b5 namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: 404e143c-6cee-4a05-931d-1c5c9ac1d547 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475c8378bb5 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 2590aee1-69db-4039-878f-65c9d5ff756a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475c8a2eed1 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: f0064830-4f70-4fc2-90ec-f3cc9280669b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475c8ac1d25 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 6707a524-d010-432e-8a47-36cedf67d5bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475cb800681 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: f48fabfa-740d-4e46-a04d-692586641f91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:32:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:23Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475cbf01b8f namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: c390ccb6-96a9-420a-abb2-52358e1da49e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:33:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:10Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68475e59f9e47 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: da0ecd21-1aee-4ad7-a72b-77e7afc2b65f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68488ea55881b namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: 030b74c4-0003-46b9-a3fb-b811a3e7f542 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 503a4b91-47f6-43c2-a877-d021c1f1ca7b kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv.18c68488ea561683 namespace: kserve-ci-e2e-test resourceVersion: "21848" uid: 220e3aaa-eeaa-4b5f-bb17-2f3622595d0d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1bf99-predictor-5c574cdd84 namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: 1d7fbffe-d9e8-4297-bb58-3be342289a79 kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'Created pod: error-404-isvc-1bf99-predictor-5c574cdd84-pdwfv' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-1bf99-predictor-5c574cdd84.18c68475500f2448 namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 75bf73eb-bc64-48a5-8845-0ef2362aaecf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: 8e5224da-5650-4407-8b94-41270bfc0b9d kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Scaled up replica set error-404-isvc-1bf99-predictor-5c574cdd84 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-1bf99-predictor.18c684754e4e4801 namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: fc0f0619-cfb3-4e80-9ff6-a0fb63c60b8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 09437a7c-c764-4ced-ab85-d8516a90ef42 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:51Z" name: error-404-isvc-1bf99-predictor.18c68478ccaeec3c namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 03238c72-e667-40e1-b0de-f42e60899062 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 09437a7c-c764-4ced-ab85-d8516a90ef42 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:51Z" name: error-404-isvc-1bf99-predictor.18c68478cd0d336b namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 03ab80c6-26fd-4bf4-bc30-f2e239ab4d81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 09437a7c-c764-4ced-ab85-d8516a90ef42 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: error-404-isvc-1bf99-predictor.18c6847fca5cb473 namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 39bc7027-0073-4874-926e-6510fbeb07c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 09437a7c-c764-4ced-ab85-d8516a90ef42 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: error-404-isvc-1bf99-predictor.18c6847fcac31e0e namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: d365a19c-07a3-46c9-964c-e7f27c16c34c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: d48c44f0-461b-4dfd-ac32-f54d682e625c kind: Event lastTimestamp: "2026-07-28T17:33:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:20Z" name: error-404-isvc-1bf99.18c684754e48f2bd namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 46b63445-4ba0-4c75-a6da-fa733efc52f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: d48c44f0-461b-4dfd-ac32-f54d682e625c kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-1bf99": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1bf99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-1bf99.18c684754ee30876 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: e9b66898-d51d-49b4-bfbf-05eeac9687c1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: d48c44f0-461b-4dfd-ac32-f54d682e625c kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1bf99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-1bf99.18c684754ee3442a namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 83fd3765-6932-4368-a41a-64c5284990e6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: d48c44f0-461b-4dfd-ac32-f54d682e625c kind: Event lastTimestamp: "2026-07-28T17:33:20Z" message: InferenceService [error-404-isvc-1bf99] is Ready metadata: creationTimestamp: "2026-07-28T17:33:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:20Z" name: error-404-isvc-1bf99.18c68482f5c7263e namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: c2dba920-222d-4f52-aeeb-df3b5fb09921 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:22:10.948381Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e71d2158ff namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: a83c3169-44c6-4351-bbea-b2d9726dcad1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73730ac07 namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 9fcc7d4d-d7b6-441d-8180-0a2a37f42758 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73897b489 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: a8ac190c-ee77-4f17-b604-c6bbeb4a2333 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73b49e70b namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: d170aedc-8d15-4e6f-bf3d-34476efaf476 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73bbeebde namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: b7cb5f50-0af4-4832-ada5-407dbd44d702 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73bc842fd namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 1df79505-5d6b-4051-bc33-08dc30da1069 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73eb474c4 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 32b71661-4694-4479-8b37-ac0923541694 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e73f24a795 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: a3c795a1-ef9b-437d-98d8-2b72e3d5699a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:31:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:27Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c683e74b989e83 namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: ac23fb0e-0ae7-42cd-8cae-32693853d51f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c6846853e85545 namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: 818c168c-37ac-4bca-b774-383e08e07a43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c6846853ebb638 namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 260b7ccf-10b4-4533-9edd-f4f3d294092d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 315213c7-2b98-4a29-9239-76401beb51f7 kind: Event lastTimestamp: "2026-07-28T17:31:27Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:27Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp.18c68468c028d5fe namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 714879c5-11ba-4dfb-99c6-726a1ca92740 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1fbfc-predictor-6c94d86f78 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 4cca8902-30b2-4a80-9f2b-2afc49bdeaa1 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: 'Created pod: error-404-isvc-1fbfc-predictor-6c94d86f78-pftbp' metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-1fbfc-predictor-6c94d86f78.18c683e71cabc406 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: f8de8431-5b94-474c-9d96-571b9caf716d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 28ef1c6b-f73d-4a1d-8d32-f4dfc9c6eab7 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Scaled up replica set error-404-isvc-1fbfc-predictor-6c94d86f78 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-1fbfc-predictor.18c683e71adbfa1c namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: e0a51fea-401a-4866-8b28-0bd32845ff38 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:22:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: cc648537-b8ba-44a1-aeb5-d2c58de05cd1 kind: Event lastTimestamp: "2026-07-28T17:22:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:40Z" name: error-404-isvc-1fbfc-predictor.18c683ea992d40ef namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: aa6e651a-77bb-45a8-92ae-9503e931bb2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:22:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: cc648537-b8ba-44a1-aeb5-d2c58de05cd1 kind: Event lastTimestamp: "2026-07-28T17:22:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:40Z" name: error-404-isvc-1fbfc-predictor.18c683ea997a29ec namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 4dcb1776-2d22-439a-bce6-814130d69d27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: cc648537-b8ba-44a1-aeb5-d2c58de05cd1 kind: Event lastTimestamp: "2026-07-28T17:23:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:25Z" name: error-404-isvc-1fbfc-predictor.18c683f196e1b203 namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: f502ee60-c9e0-4ca6-b08e-ec860bcf6015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: cc648537-b8ba-44a1-aeb5-d2c58de05cd1 kind: Event lastTimestamp: "2026-07-28T17:23:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:25Z" name: error-404-isvc-1fbfc-predictor.18c683f1973227e3 namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: dd9e5915-feda-4fc3-b2de-578808301e40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 13c6ebd4-7450-4493-b5b5-b23556a9e2ee kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:57Z" name: error-404-isvc-1fbfc.18c683e71af2f002 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 6898dd22-fdda-473c-9ef5-88be1ae0cc94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 13c6ebd4-7450-4493-b5b5-b23556a9e2ee kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-1fbfc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1fbfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-1fbfc.18c683e71b7bb336 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 98d54df4-3788-4517-9818-5fc4b6d225d9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 13c6ebd4-7450-4493-b5b5-b23556a9e2ee kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1fbfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-1fbfc.18c683e71b7bec16 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: c2b2f8e1-2809-4d17-8391-64b29d86405b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 13c6ebd4-7450-4493-b5b5-b23556a9e2ee kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: InferenceService [error-404-isvc-1fbfc] is Ready metadata: creationTimestamp: "2026-07-28T17:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:57Z" name: error-404-isvc-1fbfc.18c683f207ce40ff namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: daed19d7-a4d0-48f4-b818-bf3bcf93ec4d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:23:06.838008Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:23:06Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f4206964eb namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 8e2037fb-c975-4138-a524-3d0959624be1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3a5ae-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f42aef57dc namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 3a3e0c6d-8a95-4374-b650-6fb897b8e06e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:23:07Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f45da2e4b0 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: 6109178b-1841-4648-bb43-69f759bf0260 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f45f26bdbb namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: d420e1bc-9f09-47a2-a805-d39252427bf6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f4627251cb namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 2221d65b-b71c-4455-aca7-f127ef003965 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f462f11008 namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 89b7c074-d826-4b65-8926-fc56b28b4fd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f462fc68c6 namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: c8cd4f60-d625-4ac2-8530-ed857434ec4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:08Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f465c20ecb namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 09a4d815-9154-4463-b66a-3c26f95267d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:08Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f4663114a1 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 8902d18a-fff1-4454-81c1-315a0f315ac8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:23:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:44Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c683f49ff7c506 namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: 04659308-b981-4c4c-a7fb-4ed088ab75c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c68475482e1678 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 2fc3be54-31ad-419b-86f2-e7f1a4b16ca1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 77284c84-3f73-41eb-9ab2-352a44895984 kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:21Z" name: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55.18c68475482eb10b namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 96badce0-b900-44a9-bf29-c5507d62406d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3a5ae-predictor-7888f995f6 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: e92ac917-801b-4a75-aa34-ac7a2b1775c1 kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: 'Created pod: error-404-isvc-3a5ae-predictor-7888f995f6-fzw55' metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:06Z" name: error-404-isvc-3a5ae-predictor-7888f995f6.18c683f41ffd2102 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 266750d2-ebc0-4deb-92db-6913decbebab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 43842fc8-dc5c-4b12-b53b-ddcf7c0b55d9 kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Scaled up replica set error-404-isvc-3a5ae-predictor-7888f995f6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:06Z" name: error-404-isvc-3a5ae-predictor.18c683f41e7ac84f namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: fa2ff0e8-297e-4116-8e70-794519802326 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:23:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 47e8847e-c91b-4321-802a-12fb14eede1f kind: Event lastTimestamp: "2026-07-28T17:23:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:36Z" name: error-404-isvc-3a5ae-predictor.18c683f79d0196f3 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 6894dbdd-aad1-4dc5-b67b-4ac878e8691c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:23:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 47e8847e-c91b-4321-802a-12fb14eede1f kind: Event lastTimestamp: "2026-07-28T17:23:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:36Z" name: error-404-isvc-3a5ae-predictor.18c683f79d4e2b52 namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: 600ae137-fc70-497f-8db0-d731211af1e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 47e8847e-c91b-4321-802a-12fb14eede1f kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:06Z" name: error-404-isvc-3a5ae-predictor.18c683fe9aeb770a namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: d7e4d230-c00d-419e-b91b-ffb9360ad1a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 47e8847e-c91b-4321-802a-12fb14eede1f kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:06Z" name: error-404-isvc-3a5ae-predictor.18c683fe9b5ce477 namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: fc75a053-d927-4734-a755-2430e2f0aeed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: f0bcf1cd-d048-475d-beb0-41213edef1d5 kind: Event lastTimestamp: "2026-07-28T17:23:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:55Z" name: error-404-isvc-3a5ae.18c683f41e8fa09f namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: f4193745-34c5-461f-9481-36141b212284 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "16952" uid: f0bcf1cd-d048-475d-beb0-41213edef1d5 kind: Event lastTimestamp: "2026-07-28T17:23:55Z" message: InferenceService [error-404-isvc-3a5ae] is Ready metadata: creationTimestamp: "2026-07-28T17:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:55Z" name: error-404-isvc-3a5ae.18c683ff5e8c00f3 namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 4621edfb-a099-49a0-afba-95a9f3e51177 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:01.624969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4f50a-predictor-54cf775f94-5gqtw to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:01Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850a4f439e19 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 784ea7d2-14eb-4146-9c7d-a8abf195e92d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4f50a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850a5a7b38b8 namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: f8c1281f-a9de-4746-9cc6-b627e7bb888b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850ac296cfe6 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 1e4e2720-682f-42ea-9ff1-961503849aaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850ac454ac62 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 2777c86a-1e69-4e83-aaf8-732650d5f48c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850ac738f9ca namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 0db90d1d-0b7d-43ce-8e42-37e7714e1f8e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850ac7a36a38 namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 46d18620-e425-465f-8e49-33373f42d302 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850ac7abd70f namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: 25c1ca00-31e5-4073-ae31-b4333966b6c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850acaff1e34 namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: c935cd2a-c46a-41ff-81a9-8e72c533de23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850acb6e81c5 namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: c8c400c4-d0a3-4e5b-bded-e5ae561a94a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:43:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:49Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6850ad3169583 namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: 651daccb-0073-498b-bb44-bdb40529445d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:52:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:52:16Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6858b7994fa6f namespace: kserve-ci-e2e-test resourceVersion: "30788" uid: 544e3620-bc46-406b-ad7a-c836dcc9d594 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: fb6785bc-a8d9-4d36-b87e-df0bc221bdcc kind: Event lastTimestamp: "2026-07-28T17:52:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:52:16Z" name: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw.18c6858b799556cc namespace: kserve-ci-e2e-test resourceVersion: "30789" uid: e32cc05f-7340-4aec-b71b-93d9f7d03cf7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4f50a-predictor-54cf775f94 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 5cde38b7-98d1-4e37-b86d-6f9e02ae9451 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: 'Created pod: error-404-isvc-4f50a-predictor-54cf775f94-5gqtw' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:01Z" name: error-404-isvc-4f50a-predictor-54cf775f94.18c6850a4ec4918b namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 3394ea03-0156-4501-b038-fd363a85ce44 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: 4b81adbd-95f7-4587-b061-abb700343e15 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Scaled up replica set error-404-isvc-4f50a-predictor-54cf775f94 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:01Z" name: error-404-isvc-4f50a-predictor.18c6850a4c99f3d1 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 5f6d13bf-8271-4b1e-9d7b-ab0ee3a99d9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 65049f82-3350-4f9d-85f8-8b11f18811ef kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: error-404-isvc-4f50a-predictor.18c6850dcb1e5676 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 49f731e6-9d79-46f0-9bb6-0281198bb4e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 65049f82-3350-4f9d-85f8-8b11f18811ef kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: error-404-isvc-4f50a-predictor.18c6850dcb93c9eb namespace: kserve-ci-e2e-test resourceVersion: "27068" uid: e6ebe859-5319-4e2e-8522-1599a1a94e4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 65049f82-3350-4f9d-85f8-8b11f18811ef kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:31Z" name: error-404-isvc-4f50a-predictor.18c685114a6c4b91 namespace: kserve-ci-e2e-test resourceVersion: "27526" uid: 3343a7aa-f1ab-4855-a786-df96bd3644fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 65049f82-3350-4f9d-85f8-8b11f18811ef kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:31Z" name: error-404-isvc-4f50a-predictor.18c685114abeaa8f namespace: kserve-ci-e2e-test resourceVersion: "27527" uid: 5f0a8db9-2142-47b6-bde7-3be3a52c6b37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f50a namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: fe1e827f-84a1-40f5-9403-05d835f4c877 kind: Event lastTimestamp: "2026-07-28T17:43:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:59Z" name: error-404-isvc-4f50a.18c6850a4ca9c901 namespace: kserve-ci-e2e-test resourceVersion: "27199" uid: a02dce81-e7f6-4f04-bf2b-b8ec97111008 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: fe1e827f-84a1-40f5-9403-05d835f4c877 kind: Event lastTimestamp: "2026-07-28T17:43:59Z" message: InferenceService [error-404-isvc-4f50a] is Ready metadata: creationTimestamp: "2026-07-28T17:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:59Z" name: error-404-isvc-4f50a.18c68517e360353e namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: 8dd8020e-17aa-468c-ad2d-4b8cfcaa41eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:45.991190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c68488f0e8f5f8 namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: ca56eec8-d82e-4d16-a7ba-15012d1734ad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c684890af6e326 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: 41e74f31-2606-4fc4-aa9f-c08f1056b1ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c684890c6e84e7 namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: 80d2fee1-977e-4baf-b670-c7f2bb1d0572 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c684890f679caf namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: b5f3c2a0-e12c-4644-ab45-187c75809b7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c684890fdd2e1f namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: 958df051-0d48-4aa0-a515-f0b43105c119 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c684890fe8025c namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 2d450c63-7034-4c6a-aab6-06757ea33646 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c68489127fbbb7 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: b9616754-916e-4696-947f-ce570ccb2212 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c6848912f7f37c namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 9dd0c2ee-4e4e-4329-ae26-907fab3d4255 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c6848950d0fa6b namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: a3f251cd-8167-4c03-89d4-5a85145e2cdf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c6850a208b9c73 namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: d2646add-8281-4b8b-88be-6275e10be994 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c6850a208c1d9d namespace: kserve-ci-e2e-test resourceVersion: "26442" uid: 113d1446-0526-470a-a5fa-1d5330111f73 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a935516e-f8e9-47d4-94a2-c90e00577ddb kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2.18c6850ac54ba78f namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 619e2f23-c7c4-4a53-965f-94c561690189 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9d7bb-predictor-85c9b46cf4 namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: c3cf7bbb-0ed4-4fdc-8abb-d62a1f005bc5 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: 'Created pod: error-404-isvc-9d7bb-predictor-85c9b46cf4-wzht2' metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-9d7bb-predictor-85c9b46cf4.18c68488f09023bd namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 5c9ac899-d1a1-4b98-aa0f-72727664a895 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: 0eff1f7f-db3d-486d-93e1-27b4e2504c41 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Scaled up replica set error-404-isvc-9d7bb-predictor-85c9b46cf4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-9d7bb-predictor.18c68488ef0ebc6b namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 4a024117-045a-4899-945b-b3f8b6f8691d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: 97d679c7-21f9-4c67-b801-3e4496879e41 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:31Z" name: error-404-isvc-9d7bb-predictor.18c6848c6dbd6a17 namespace: kserve-ci-e2e-test resourceVersion: "22281" uid: 2933c78e-d85c-4f02-9af7-0bfcb97f7894 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: 97d679c7-21f9-4c67-b801-3e4496879e41 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:31Z" name: error-404-isvc-9d7bb-predictor.18c6848c6e4bf8d3 namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: 615a507e-e39b-454c-81ca-113a88e2cad7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: 97d679c7-21f9-4c67-b801-3e4496879e41 kind: Event lastTimestamp: "2026-07-28T17:35:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:16Z" name: error-404-isvc-9d7bb-predictor.18c68496ea52a408 namespace: kserve-ci-e2e-test resourceVersion: "22660" uid: 0bab412d-40f0-4889-a004-29bcfc888e84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: 97d679c7-21f9-4c67-b801-3e4496879e41 kind: Event lastTimestamp: "2026-07-28T17:35:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:16Z" name: error-404-isvc-9d7bb-predictor.18c68496eab434fd namespace: kserve-ci-e2e-test resourceVersion: "22662" uid: cb357eb0-31e8-46ea-8855-e492aa2e68fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: 63cfeae9-bf50-4c63-a668-68de8c9a8813 kind: Event lastTimestamp: "2026-07-28T17:34:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:33Z" name: error-404-isvc-9d7bb.18c68488ef1678e7 namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: 711c2916-9e42-47c8-a58d-fc36e5d78993 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: 63cfeae9-bf50-4c63-a668-68de8c9a8813 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-9d7bb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9d7bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-9d7bb.18c68488ef8f2a98 namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: af277231-3738-4539-8bbe-3f6258108bf0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: 63cfeae9-bf50-4c63-a668-68de8c9a8813 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9d7bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:45Z" name: error-404-isvc-9d7bb.18c68488ef8f7dfa namespace: kserve-ci-e2e-test resourceVersion: "21876" uid: 29ec5816-c06f-4af8-bb02-076869da923c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: 63cfeae9-bf50-4c63-a668-68de8c9a8813 kind: Event lastTimestamp: "2026-07-28T17:34:33Z" message: InferenceService [error-404-isvc-9d7bb] is Ready metadata: creationTimestamp: "2026-07-28T17:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:33Z" name: error-404-isvc-9d7bb.18c684940ced4c19 namespace: kserve-ci-e2e-test resourceVersion: "22314" uid: 0d53db6a-8ddb-4521-8d6e-934e352f7e94 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:31:25.970216Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b0164-predictor-dd9648568-nwtkk to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6846857028608 namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: f5f2edd0-b66d-4203-a8e0-afc8c57a529c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6846870cd08bb namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 3e387f36-120f-4f89-aac7-b617ce3bee60 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c68468722e809b namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 0c36bd75-65f1-484b-b361-f620f141b4e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c68468753e9626 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: e1713dcb-cf43-4c95-8fcb-aff726b00254 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6846875af4b98 namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 04f7f31d-c46f-4919-a9ba-ca20df4cca83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6846875b95d7b namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: eb15e437-9a9e-4150-a7f8-9ebc551c1fc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c68468784921ab namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: f006a6c9-0622-44b6-a673-45a6a52bf133 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6846878b3f4c0 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: ae88f1be-ff69-4622-b302-243a39793768 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:32:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:02Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6846890a4b49a namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: a564e4f4-beee-4f4d-b880-0436d0d12e39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c68478acc1b2fd namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 3462da62-1f41-4579-a0e8-dacb3a19f739 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c68478acc2317b namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 9fac87ba-24c5-4ab2-9bbc-6c1b6283b80e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b0164-predictor-dd9648568-nwtkk namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 46d8f25f-49af-4e5c-96cf-fef30bc44530 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-b0164-predictor-dd9648568-nwtkk.18c6847918fa194b namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: 837b7536-f405-43bd-9020-f1f499d400f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b0164-predictor-dd9648568 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: d25f15c7-fbc9-43b8-8241-905333e9957c kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: 'Created pod: error-404-isvc-b0164-predictor-dd9648568-nwtkk' metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-b0164-predictor-dd9648568.18c68468569147ea namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 38acffcb-6306-441b-adf9-883dd4c64774 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: 43433f0b-e2fd-4e86-b087-7fa03649b5c5 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Scaled up replica set error-404-isvc-b0164-predictor-dd9648568 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-b0164-predictor.18c6846854d2bfb1 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: 8d9028a8-1bd8-4c94-89e7-bcfd1fdfe576 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 59431451-ae62-4f86-8af9-8487cccf5f39 kind: Event lastTimestamp: "2026-07-28T17:31:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:40Z" name: error-404-isvc-b0164-predictor.18c6846bd33c2540 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 8f9e1f69-1912-4bc7-8a8b-c75a8bb753ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 59431451-ae62-4f86-8af9-8487cccf5f39 kind: Event lastTimestamp: "2026-07-28T17:31:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:40Z" name: error-404-isvc-b0164-predictor.18c6846bd3a0ed74 namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: 54f3e476-3eb2-4a75-8fd8-28c3a094988f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:31:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 59431451-ae62-4f86-8af9-8487cccf5f39 kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:31:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:25Z" name: error-404-isvc-b0164-predictor.18c6846f527b2372 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 7ea222fa-b4c0-47c7-86db-01746a11b6a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:31:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 59431451-ae62-4f86-8af9-8487cccf5f39 kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:31:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:25Z" name: error-404-isvc-b0164-predictor.18c6846f52edb14c namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 6b80974d-d9d5-4949-941e-9f360addcecf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 1babcb52-4445-4962-95a8-b7a02e0d03ab kind: Event lastTimestamp: "2026-07-28T17:32:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:13Z" name: error-404-isvc-b0164.18c6846854fc6056 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 7023ea5d-67bb-416e-8710-bfd43d78390f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 1babcb52-4445-4962-95a8-b7a02e0d03ab kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-b0164": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b0164": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-b0164.18c68468557a04a7 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: d3f5cd3b-a326-4480-9daf-148be6b14f36 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 1babcb52-4445-4962-95a8-b7a02e0d03ab kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b0164": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:31:25Z" name: error-404-isvc-b0164.18c68468557a3966 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 2c90344b-4219-497e-b122-078ac3bac008 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 1babcb52-4445-4962-95a8-b7a02e0d03ab kind: Event lastTimestamp: "2026-07-28T17:32:13Z" message: InferenceService [error-404-isvc-b0164] is Ready metadata: creationTimestamp: "2026-07-28T17:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:13Z" name: error-404-isvc-b0164.18c684734c681773 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: dc65cc31-7c16-4155-b27a-568f403040fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:41:51.193949Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b5d39-predictor-84ff846864-69s5t to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684f9e93f9053 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 26881a65-8983-49fe-8212-8e956d894e33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25756" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa030805d7 namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: f2052fe3-4155-43e2-8f50-c197179c1761 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa048dced0 namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 49968fa7-7dc5-4463-9ae7-1faf52348c2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa080df1c4 namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: bd006c06-61ed-40ac-abc4-306ceb63b5f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa0879818b namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: bafa474f-83be-48d8-a1d8-ca57a352d103 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa0883664e namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: 1d0f7311-ba9f-4d53-93ae-07cdb59d7316 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa0bf1df8c namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: e78849a0-b1c6-416f-950f-c35938ff42ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa0c638424 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: d17005ed-3069-40ee-bb2a-b70a16f4a3d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:42:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:28Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c684fa37fa4851 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 5f2b89cc-d324-4e23-ae7d-5ecd48fb81d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c6850a52aa71c2 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 5a3eefde-a75c-4a08-8f30-0e7de0d958d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c6850a52aae35a namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 0c4ff778-affb-4e96-b785-359829e5782a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5d39-predictor-84ff846864-69s5t namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: b2e48201-4e75-41a0-85b3-f5088eac37cb kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: error-404-isvc-b5d39-predictor-84ff846864-69s5t.18c6850ac0472048 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: f7a6a03e-d4b7-4dff-88ec-8c0fc317e786 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b5d39-predictor-84ff846864 namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: c45b75cd-54c9-46fc-9123-aeeda857857f kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Created pod: error-404-isvc-b5d39-predictor-84ff846864-69s5t' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor-84ff846864.18c684f9e8cdc8f7 namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: 1771df15-a27e-4619-b1fb-237ca76824ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: 2be4d3cd-ff26-44e2-a0ec-11346f6ed76b kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Scaled up replica set error-404-isvc-b5d39-predictor-84ff846864 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39-predictor.18c684f9e6f3cc0a namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: b0122031-7870-4c88-8bad-b072a69be170 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:42:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: 8dbe5c3a-b961-4107-80a8-85232ba16906 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:51Z" name: error-404-isvc-b5d39-predictor.18c684fd658dfebf namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 4a57aab8-b4ab-42a7-8826-4605ceeec444 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:42:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: 8dbe5c3a-b961-4107-80a8-85232ba16906 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:51Z" name: error-404-isvc-b5d39-predictor.18c684fd65e37ae8 namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: c273495a-6444-45f5-837c-b85ccd26c1c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 8dbe5c3a-b961-4107-80a8-85232ba16906 kind: Event lastTimestamp: "2026-07-28T17:42:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:21Z" name: error-404-isvc-b5d39-predictor.18c68500e5956bd8 namespace: kserve-ci-e2e-test resourceVersion: "26029" uid: 5294b052-817e-4b63-a105-9fd6916f9c3e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 8dbe5c3a-b961-4107-80a8-85232ba16906 kind: Event lastTimestamp: "2026-07-28T17:42:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:21Z" name: error-404-isvc-b5d39-predictor.18c68500e6165d5f namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: 743383bc-3075-4218-a5c7-efd3c10e88a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: af371d8e-6866-4c95-8d3a-c1886613f033 kind: Event lastTimestamp: "2026-07-28T17:42:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:38Z" name: error-404-isvc-b5d39.18c684f9e70ad0b3 namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 84a4cb44-9742-4ed1-b5a7-b561d4253a58 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: af371d8e-6866-4c95-8d3a-c1886613f033 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-b5d39": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b5d39": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39.18c684f9e7c6fa48 namespace: kserve-ci-e2e-test resourceVersion: "25744" uid: 6238e565-011a-4f0a-b384-b832b9c89c91 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: af371d8e-6866-4c95-8d3a-c1886613f033 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b5d39": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-b5d39.18c684f9e7c741bb namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: 3984b145-8da3-4ec8-8527-8e7d4b42a93e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: af371d8e-6866-4c95-8d3a-c1886613f033 kind: Event lastTimestamp: "2026-07-28T17:42:38Z" message: InferenceService [error-404-isvc-b5d39] is Ready metadata: creationTimestamp: "2026-07-28T17:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:38Z" name: error-404-isvc-b5d39.18c68504f419a009 namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 4e1014aa-49d0-4732-a475-ff2185fc950d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:20:36.867680Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:20:36Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d1357bba35 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: a77a252e-8288-40ab-9902-995f02448224 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d0cd1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d13fb3a078 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: ce1ff1ed-92dc-4007-9709-8f0e099e74b5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:20:37Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d17664579a namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: af203fc9-e3a6-4fa3-bfa6-c56592d45371 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d178744e03 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: e2ee1a6a-bb2c-445a-bbe7-0eafe3892d92 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" in 16.589s (16.589s including waiting). Image size: 1333940313 bytes.' metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d55540ab6e namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 460abdfe-c6c9-4621-a4e7-5960a2939752 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d559b77317 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: c8ae886d-393c-418e-a418-0196a35f4888 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d55a363495 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 51a718d1-773e-49b8-b146-ec8d51754279 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d55a43b857 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 3ecaeca4-089c-4292-8997-f80651dd8f38 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.642s (2.642s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:57Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d5f7cc23a8 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 6584c01d-38f3-4618-9e99-ff7ff6371e05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:57Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d5fb690b08 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: a0f3f9c6-69eb-4ef7-9d4a-9f926864005f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:20:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:57Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d5fc13a118 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: bccff2a4-e7ff-4d45-a98b-0e4ac80f6545 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:22:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:13Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683d5fe5c875f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 8c7931a8-b0d9-41fe-b7f5-53bd563f5779 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683e71bde56ec namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 01392afd-b6dc-48b3-a741-e722f687fc4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:10Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683e71bdecbea namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 54be901b-bbe2-4e6a-adab-fafed4070134 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 53910743-7650-4424-8664-eacac1167e83 kind: Event lastTimestamp: "2026-07-28T17:22:13Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:13Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv.18c683e7b0bd206a namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 69d8df50-2e83-4011-9fee-2e9acb718b51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d0cd1-predictor-6d8bd88894 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 3c7245c9-8735-4cef-9a99-162a371845bf kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'Created pod: error-404-isvc-d0cd1-predictor-6d8bd88894-j7lhv' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: error-404-isvc-d0cd1-predictor-6d8bd88894.18c683d13511ed91 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: d439854d-200b-4317-878c-855756291f86 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: 597b68d5-364f-4e34-9451-54b8cf64186b kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Scaled up replica set error-404-isvc-d0cd1-predictor-6d8bd88894 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: error-404-isvc-d0cd1-predictor.18c683d131d91ae9 namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 2656f8e3-3be4-4318-9fdf-d67f56fe1fd6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 233d774b-687e-41f5-9677-05a514a6a370 kind: Event lastTimestamp: "2026-07-28T17:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:21Z" name: error-404-isvc-d0cd1-predictor.18c683d4b064e003 namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: ada3dcc0-6f90-4c2a-93fc-4ac9c1bca73c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 233d774b-687e-41f5-9677-05a514a6a370 kind: Event lastTimestamp: "2026-07-28T17:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:21Z" name: error-404-isvc-d0cd1-predictor.18c683d4b0d7e999 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 7f5b19a8-6072-4335-b3e1-6e3d074ee43c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 233d774b-687e-41f5-9677-05a514a6a370 kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:06Z" name: error-404-isvc-d0cd1-predictor.18c683df2d58d0d4 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 62e4b83b-57fa-4b37-bf51-7e8b5253307f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 233d774b-687e-41f5-9677-05a514a6a370 kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:06Z" name: error-404-isvc-d0cd1-predictor.18c683df2db1a4f3 namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: b311978f-6679-48d5-be4c-f1e972489ba8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0cd1 namespace: kserve-ci-e2e-test resourceVersion: "14693" uid: 251e5dc7-1c1d-44db-96b2-854d8584a46a kind: Event lastTimestamp: "2026-07-28T17:21:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:43Z" name: error-404-isvc-d0cd1.18c683d131f3d0a6 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 30169407-456b-4ad1-822d-d781e165ccfc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0cd1 namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: 251e5dc7-1c1d-44db-96b2-854d8584a46a kind: Event lastTimestamp: "2026-07-28T17:21:43Z" message: InferenceService [error-404-isvc-d0cd1] is Ready metadata: creationTimestamp: "2026-07-28T17:21:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:43Z" name: error-404-isvc-d0cd1.18c683e0ba887de2 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: a2f902b9-aa31-4453-8813-c5a10533476e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:00.988578Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a29551260 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 844680e5-3d66-4873-8499-6ff9b01db02f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e4209-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a36663793 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: a91bfdf9-f123-4a89-bc45-194609dda0f0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a66993f8f namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: 56f808a8-f17b-47ae-83f6-18559151855d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a6814dd80 namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: ba3dc447-384b-40bf-b57d-a40acf9ac0ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a6b79ee1e namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 3033b0b1-3ae2-46a2-8fe8-cc738c2bf34f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a6beaa339 namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: 9d6b2e0b-5b4b-444d-b7a1-543c59682b2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a6bf3bedb namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 8cc6dce2-4d75-4faa-ab70-8cc553c16fad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a6f1edb26 namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: e0e8da2f-9bf7-451c-b68b-6070bf2334de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850a6fc5aee2 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: e399cf27-9d15-4342-9387-d16b7beee3a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T17:43:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:39Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c6850ad31b14ce namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 8fad4345-8110-443e-b0d3-a6c5f731e312 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T18:00:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:00:30Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c685fe8a338583 namespace: kserve-ci-e2e-test resourceVersion: "34210" uid: 5677fb44-e1c1-45e8-b8f7-582cc31213b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 22f17563-00e2-47d2-91e8-662f3833f2f6 kind: Event lastTimestamp: "2026-07-28T18:00:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:00:30Z" name: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4.18c685fe8a3430cf namespace: kserve-ci-e2e-test resourceVersion: "34211" uid: e62bcb7c-b11d-40a7-b8f0-d78ae11ce094 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e4209-predictor-595c5d94bb namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 43d33acd-f5ac-49ee-ad79-377bde7a9af3 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: 'Created pod: error-404-isvc-e4209-predictor-595c5d94bb-tg9l4' metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-e4209-predictor-595c5d94bb.18c6850a28ce6879 namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: e6c4e50d-6f6a-4581-bad9-c56e2308107a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: 7a3f3423-10be-411c-ba05-3bcfe1b9e385 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Scaled up replica set error-404-isvc-e4209-predictor-595c5d94bb from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-e4209-predictor.18c6850a26f0141f namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: b48d6e63-8df0-4e3c-8b22-add7139beae2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: 29a496fc-c44d-4e7c-ad1d-77084987eab4 kind: Event lastTimestamp: "2026-07-28T17:43:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:30Z" name: error-404-isvc-e4209-predictor.18c6850da56743b7 namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: 6b805845-1cf9-4342-892f-fb56c56718f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: 29a496fc-c44d-4e7c-ad1d-77084987eab4 kind: Event lastTimestamp: "2026-07-28T17:43:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:30Z" name: error-404-isvc-e4209-predictor.18c6850da5de6a93 namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: 8a39b7cb-fe9b-4a47-8a04-884353d5a52e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:43:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26803" uid: 29a496fc-c44d-4e7c-ad1d-77084987eab4 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:31Z" name: error-404-isvc-e4209-predictor.18c68514a3a22e46 namespace: kserve-ci-e2e-test resourceVersion: "27516" uid: 707eb286-d789-415e-8506-d48e9d37e03f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:43:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26803" uid: 29a496fc-c44d-4e7c-ad1d-77084987eab4 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:31Z" name: error-404-isvc-e4209-predictor.18c68514a41582dd namespace: kserve-ci-e2e-test resourceVersion: "27517" uid: 657b6ee8-c425-43d4-ba45-7e59235233cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 2cbd2a78-6da1-4724-9e7e-3ca40f874d84 kind: Event lastTimestamp: "2026-07-28T17:43:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:50Z" name: error-404-isvc-e4209.18c6850a26f86c7f namespace: kserve-ci-e2e-test resourceVersion: "27110" uid: 86989a8b-1adc-45d2-894c-d71e3cc12e1b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 2cbd2a78-6da1-4724-9e7e-3ca40f874d84 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-e4209": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4209": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-e4209.18c6850a27833b0a namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: c79ce309-5503-4c88-b109-cc8fe68118bb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 2cbd2a78-6da1-4724-9e7e-3ca40f874d84 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4209": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:00Z" name: error-404-isvc-e4209.18c6850a27837f12 namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 6fc4ca74-10cd-4774-9053-3129298054cf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: 2cbd2a78-6da1-4724-9e7e-3ca40f874d84 kind: Event lastTimestamp: "2026-07-28T17:43:49Z" message: InferenceService [error-404-isvc-e4209] is Ready metadata: creationTimestamp: "2026-07-28T17:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:49Z" name: error-404-isvc-e4209.18c685158f5618ce namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: 6b7608f2-2056-41d8-acd6-914d9ceec17e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:36.241393Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fca3b-predictor-678fd4d868-5gwnh to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478b37f778f namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 0735ed33-412d-42a5-bb13-e2ff7592b1c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fca3b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478bcb7a16e namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 81a8d482-a3ba-458d-ac38-f1eab9c669b2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478f19addde namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 8539442e-c39e-4395-aac5-ef6ecaf2dff0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478f2fd2803 namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: 0c8ad5a2-13e8-46a4-99f9-5cdfac82a46b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478f5ff5d0b namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 1df5a04e-978d-45ae-b128-49cd65c182bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478f676021b namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 5b006cf4-7c60-4966-b4ab-7d3036e9460c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478f681b0c6 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 113991c0-0fd8-43cf-8061-f0ab8e54dac5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478fa6501bb namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 3615c03f-fd5b-4ae0-901f-a2f64a1f1147 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c68478fad3b974 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: dee85d65-1718-445e-8c17-586bb6558bc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:33:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:25Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c6847969063a9a namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: ee1137f7-4534-41ec-8ce8-1cfcbaf76bf6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c684f9e72f183b namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: d92909e4-607c-42d2-90bb-668976348f77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a0ac4ed0-512e-4fe0-a7fa-f17be9ba2af9 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh.18c684f9e72f9103 namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 2cc31568-f65b-400b-be68-98e78047beb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fca3b-predictor-678fd4d868 namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: be5c3b86-c664-4fcc-9c58-d390bc31b7cc kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: 'Created pod: error-404-isvc-fca3b-predictor-678fd4d868-5gwnh' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-fca3b-predictor-678fd4d868.18c68478b2c30b74 namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: a579d9bf-0d19-4926-b134-3057920b95b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 82ed2de4-f5f5-4740-ad44-5dca37bde809 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: Scaled up replica set error-404-isvc-fca3b-predictor-678fd4d868 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-fca3b-predictor.18c68478b0e0786b namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: c2a684ad-89aa-4d90-8684-fd5900a22e5f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 07ee1939-2416-42c6-a773-c15ad6a258ea kind: Event lastTimestamp: "2026-07-28T17:34:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:06Z" name: error-404-isvc-fca3b-predictor.18c6847c2fd38c15 namespace: kserve-ci-e2e-test resourceVersion: "22095" uid: 546c28bc-85ab-449d-ba7c-8ea9dcdb9361 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 07ee1939-2416-42c6-a773-c15ad6a258ea kind: Event lastTimestamp: "2026-07-28T17:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:06Z" name: error-404-isvc-fca3b-predictor.18c6847c30365fb0 namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: ea94e7e2-6a4f-49fc-b0e8-73345e1a1aa2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 07ee1939-2416-42c6-a773-c15ad6a258ea kind: Event lastTimestamp: "2026-07-28T17:33:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:06Z" name: error-404-isvc-fca3b-predictor.18c6847faf78c67b namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: b8a3d6db-ec92-435a-a759-e190b7e53d91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 07ee1939-2416-42c6-a773-c15ad6a258ea kind: Event lastTimestamp: "2026-07-28T17:33:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:06Z" name: error-404-isvc-fca3b-predictor.18c6847fb00b4558 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 2c0dc1d2-d131-4436-97f6-8887c6bf2f4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 53024375-3195-4317-ab53-439585062667 kind: Event lastTimestamp: "2026-07-28T17:33:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:35Z" name: error-404-isvc-fca3b.18c68478b0e70fb8 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: 15243991-7cc8-43c3-8ad7-994f7446fd05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 53024375-3195-4317-ab53-439585062667 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-fca3b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fca3b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-fca3b.18c68478b172687a namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 4b173d6f-fc19-4d85-86b4-33fe9c4f3254 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 53024375-3195-4317-ab53-439585062667 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fca3b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:36Z" name: error-404-isvc-fca3b.18c68478b172de4c namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 826609a3-1751-445b-a7c5-d5f13abea6d5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 53024375-3195-4317-ab53-439585062667 kind: Event lastTimestamp: "2026-07-28T17:33:35Z" message: InferenceService [error-404-isvc-fca3b] is Ready metadata: creationTimestamp: "2026-07-28T17:33:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:35Z" name: error-404-isvc-fca3b.18c6848678f9a66c namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: b9a28a72-ad33-4ffb-b22e-07abb26e181c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:20:36.481428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj to ip-10-0-139-193.ec2.internal metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d11e75fef9 namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 8895e097-23b3-45a5-b887-ea341f8cb2d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d139b0b05a namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: c8a81dd8-6b82-4548-923f-5bacee8cab6a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d13b50b260 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 378b166d-c171-4523-a4d3-db3892449677 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d13eb15bc4 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: bddacd3b-b1ff-404e-9f0f-0b71dfe7b526 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d13f23dcdb namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 45aecbf2-4dc2-4984-a44a-a452a84dda84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:20:41Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:41Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d24ac03d6b namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: 645bf0f5-a949-4c54-8b0d-7075401bb8e3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 24.742s (24.742s including waiting). Image size: 1555722147 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d80d88fe06 namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: bca46c47-b084-4ef1-8995-4b6f84c098e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d813f7426e namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: 9c3bfab7-8729-4a28-84d9-944e9ac863dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d814c0286a namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 0ebee498-d8ff-466e-8d22-13847afe3ddb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d81e34d612 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: c10bba3c-0796-4e1a-b4d5-11d8465a4f60 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:09Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.139s (3.139s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:09Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d8d951900e namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 09fc4188-5ad1-428d-9d20-ee36fc8de01b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:09Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d8e5627777 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: bf12560d-b5b5-403a-95ba-114769f837db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:21:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:09Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d8e7063e32 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: ac332da4-58aa-459d-b88a-1c4f4692b6f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-28T17:21:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:23:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:10Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683d9514efc18 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 3248c5b4-162d-462c-b4f0-5115adf71417 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683f419ca084f namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 2cf64d5f-79bc-4408-a284-9649a0cf1f9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683f419cd57bb namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 5ba3ad46-12bd-4e0f-a7e9-b7319bdb86ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: c369e90f-8cba-427f-8dac-d0669e473abe kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj.18c683f453a12c22 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: b9af4c5d-20a5-44f7-b642-a6a46cfe6dd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84c58bd689 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 5803cfd2-1e2f-4afa-bf02-dc9377b458c6 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84c58bd689-pmmlj' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1-predictor-84c58bd689.18c683d11dd07fd4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: afc354b1-0ebb-4992-97fb-cc6f850b73da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 093d1ba0-4263-465e-bd00-15ee3496aaf8 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84c58bd689 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1-predictor.18c683d1167efd8c namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 8a4322be-4e1b-48e5-8673-1d17f6fb673f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: dae0f749-2a4e-4bf0-921f-9847dcba6f12 kind: Event lastTimestamp: "2026-07-28T17:21:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:36Z" name: isvc-sklearn-graph-1-predictor.18c683d49c49632b namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 400f09c4-03f7-47bf-ae6f-a814083278c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: dae0f749-2a4e-4bf0-921f-9847dcba6f12 kind: Event lastTimestamp: "2026-07-28T17:21:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:36Z" name: isvc-sklearn-graph-1-predictor.18c683d49cd0270a namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 61d89446-1e41-4ead-baa2-261c0bb551dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: dae0f749-2a4e-4bf0-921f-9847dcba6f12 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:51Z" name: isvc-sklearn-graph-1-predictor.18c683e29a2de5f4 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 1b1d62ca-c5d4-4d18-b713-5c8ee95c7da7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: dae0f749-2a4e-4bf0-921f-9847dcba6f12 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:51Z" name: isvc-sklearn-graph-1-predictor.18c683e29abd1605 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 0cca0a1e-bc80-42c2-8020-6482ec2ccf86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: d7598ae0-3673-44a2-b28d-9eef9f1ee5d2 kind: Event lastTimestamp: "2026-07-28T17:22:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:31Z" name: isvc-sklearn-graph-1.18c683d11c86dd28 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 866beebb-bb33-4154-8ce8-e5988621a165 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: d7598ae0-3673-44a2-b28d-9eef9f1ee5d2 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1.18c683d11cfae229 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: abf60146-1109-46d5-a7e6-b1368d9908f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: d7598ae0-3673-44a2-b28d-9eef9f1ee5d2 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-1.18c683d11cfb2337 namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 6ee31d48-fbf5-4c3a-92f5-5197df8f0903 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: d7598ae0-3673-44a2-b28d-9eef9f1ee5d2 kind: Event lastTimestamp: "2026-07-28T17:22:31Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T17:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:31Z" name: isvc-sklearn-graph-1.18c683ebcac9f85f namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: aa9be288-37e2-40e0-ba82-382bdd1f797a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:20:36.979134Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d13c206d89 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 5d163250-1bab-498c-91f6-1dc29a91737d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d145b336dc namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: b1c20255-8342-4226-909e-e839bbdfdd68 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:39Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:20:39Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d1b4fd9729 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 216a1323-998f-4bb7-9a79-18f2c651a8d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:39Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" metadata: creationTimestamp: "2026-07-28T17:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:39Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d1b739cbe1 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 6610ac3a-bec4-4942-bc07-04610ff61adc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" in 15.462s (15.462s including waiting). Image size: 300977080 bytes.' metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d550e05384 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: efc90256-61a4-43c6-9060-b55ee5eef64b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d5542c73cc namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 0179cafb-3ba8-4b03-9506-c5885b7a387a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d554a0bbf2 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: cd566a62-fb52-405e-b0d0-5a2761436d69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:20:58Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:58Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d63a6de855 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 68280112-8054-4d76-afbb-7bae0cde5fde reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:21:04Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 5.929s (5.929s including waiting). Image size: 1555722147 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d79be013d0 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 68a27fd4-b0a9-4f6f-b5fb-f488d90eb59f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:21:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d79f78058d namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 499947e2-a286-49ae-8d89-22ae27d753c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:21:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d79fe8e624 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: fe3a2ee7-e72e-4deb-a911-fd1deb55bac4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:21:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d7a0031b7b namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: ccf173b7-abd5-4e1a-8a69-77b0f4a49244 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:21:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d7a332cc3f namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 14bc0724-22ec-4cf5-8b82-92016aa4c210 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:21:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d7a3a2d2a8 namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: 0da760b2-5193-483d-8755-b9f14301077e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:22:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:12Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683d8198a23f7 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: ecbccf08-738f-42a9-9a3a-82279a49eee3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683f4161dc036 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: c3dce639-5c96-4bcd-b6db-57b0d5ea1927 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683f4161e2dbd namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 86950ec9-a883-4edc-a857-785e54a6d59c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 10241291-86e0-4bb5-93b5-008f308661e2 kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7.18c683f445e94bed namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 81c271bb-7e1a-489b-84ff-9da3a7200e6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5bc9545b47 namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 3d311050-5d43-4c34-a2c2-266f81cde458 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5bc9545b47-jvsx7' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47.18c683d13b9772bc namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 4bb415b9-4e86-416a-90bb-000fda69d02d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: c03be81d-0198-4121-a3ae-c18f012dfcdd kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5bc9545b47 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-sklearn-graph-2-predictor.18c683d138a2507f namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 4f355483-25b8-4f48-af0d-4a162ea0c693 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: b3237a45-674f-4cf2-82e2-1dcf31661227 kind: Event lastTimestamp: "2026-07-28T17:21:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:36Z" name: isvc-sklearn-graph-2-predictor.18c683d4b7688fa1 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: 36c8feca-4504-4863-a90b-6682ae0a4c23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: b3237a45-674f-4cf2-82e2-1dcf31661227 kind: Event lastTimestamp: "2026-07-28T17:21:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:36Z" name: isvc-sklearn-graph-2-predictor.18c683d4b7e21b64 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 7a71dcbe-6ace-4ebd-927c-855933a43c2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: b3237a45-674f-4cf2-82e2-1dcf31661227 kind: Event lastTimestamp: "2026-07-28T17:22:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:37Z" name: isvc-sklearn-graph-2-predictor.18c683dbb573c97f namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 5808e8a2-a1ab-49b7-b083-9ed0105d3f54 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: b3237a45-674f-4cf2-82e2-1dcf31661227 kind: Event lastTimestamp: "2026-07-28T17:22:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:37Z" name: isvc-sklearn-graph-2-predictor.18c683dbb5e028e9 namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: b1191443-f034-482f-9250-98b90c156f65 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 81948c3c-48eb-4946-a469-0a67966bcde6 kind: Event lastTimestamp: "2026-07-28T17:22:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:22Z" name: isvc-sklearn-graph-2.18c683d138bb57d2 namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 16fb55f1-d25a-4a1a-889b-a79ca58eea89 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 81948c3c-48eb-4946-a469-0a67966bcde6 kind: Event lastTimestamp: "2026-07-28T17:22:22Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T17:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:22Z" name: isvc-sklearn-graph-2.18c683e9d2227b79 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 96947afc-1797-475d-95c6-e1bd884285da reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:20:36.718276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-ghf5z to ip-10-0-139-193.ec2.internal metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:20:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d12c9403be namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 9b361641-4aa2-493f-b237-d4730b9850e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d13568b084 namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 1ae15480-feb5-44ec-9c08-195990d31574 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:20:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d169e55b75 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: d870008a-c6b5-4f5f-a394-eb8241ad9ffd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d16b61779b namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: cce01d2b-eb96-45db-aa98-084bb6e53a8a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d16f8e078d namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 3767055c-9315-4885-95d4-6635e4bcb382 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d16ff6ea6b namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: d5d21c95-b376-4b86-89db-23ee08561ebc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:20:42Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T17:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d28735a7b1 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 1a7f5fee-a0b2-4c17-af80-30f7e578ff83 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.074s (24.074s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d82229be2e namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 6a001e98-1acf-466e-a7c3-1912935ba06a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d8282ef7ef namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 2bbb6e18-05ea-454c-a937-c61a408310ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d828d2b945 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 93522cc7-87c1-41ac-b102-c7a5a33e6901 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:06Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d828e05f90 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: e42638fa-d57d-49bf-add9-79e983edc232 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:09Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.094s (3.094s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d8e159bdcc namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: c911a826-304f-4d30-bb31-aec7dc693b21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d8e7dfafc8 namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 640d0526-c505-4b65-a138-183b241f326c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:21:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d8e8f32676 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: a0141eb2-36f5-4275-baac-e1b6ce0fc834 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T17:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683d915734759 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: f39fded8-5d8a-43aa-bf8c-6c3a972439bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683f41cf31190 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 41f87925-ffc7-4e91-890f-77b7a642eecc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: ebe8c668-bf03-4eb4-9e6d-5b89f3af8f8e kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-ghf5z.18c683f41cf39191 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 4c78a2f0-566c-41f7-b118-2c53d58c3ab3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-193.ec2.internal source: component: kubelet host: ip-10-0-139-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 4932c30f-52ee-4f01-a23f-e96eb98e0755 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-ghf5z' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c683d12c062865 namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: 24841dd2-523f-4bc1-bc5a-7c60d800148b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: f8c86633-3d38-45a6-bffe-84fb01564392 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: isvc-xgboost-graph-predictor.18c683d1291de4b1 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 926d22ff-ed4b-4383-8fa5-7d11c64be11a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 8596523b-720d-44de-8f5f-6b4fc1c9d495 kind: Event lastTimestamp: "2026-07-28T17:21:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:36Z" name: isvc-xgboost-graph-predictor.18c683d4a7b26e08 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: b5d5de3d-692e-4d10-8330-49cad796bb8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 8596523b-720d-44de-8f5f-6b4fc1c9d495 kind: Event lastTimestamp: "2026-07-28T17:21:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:36Z" name: isvc-xgboost-graph-predictor.18c683d4a8123e1f namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: b7fb3f22-28e6-4dda-9c6d-b2cce42cf163 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 8596523b-720d-44de-8f5f-6b4fc1c9d495 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:51Z" name: isvc-xgboost-graph-predictor.18c683e2a335aa67 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 55356f30-4b5c-44c1-9b85-88e2918d442f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 8596523b-720d-44de-8f5f-6b4fc1c9d495 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:51Z" name: isvc-xgboost-graph-predictor.18c683e2a3b1ee0c namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 8a3e9afd-c97f-4182-953a-3aaae3a36baa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 3afa545a-813f-4c46-bce8-9faee9865a60 kind: Event lastTimestamp: "2026-07-28T17:22:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:16Z" name: isvc-xgboost-graph.18c683d12921efc2 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: c7dc95eb-b23f-444c-b59a-44a662e6ddf1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 3afa545a-813f-4c46-bce8-9faee9865a60 kind: Event lastTimestamp: "2026-07-28T17:22:16Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T17:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:16Z" name: isvc-xgboost-graph.18c683e879daee8a namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: d416b848-e6b6-40d4-b54c-ae7255caf641 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:22:56.431055Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-64698579b9-p8c24 to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:22:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:22:56Z" name: model-chainer-64698579b9-p8c24.18c683f1b41be1f9 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: b1dc76ca-546c-4457-8cb3-3ec651a85c4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:56Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:22:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:56Z" name: model-chainer-64698579b9-p8c24.18c683f1bf13b9b3 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: d2d0b54e-61d0-478e-88db-bb1fee3561cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:57Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:22:57Z" name: model-chainer-64698579b9-p8c24.18c683f1f17e167d namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 913360b7-1fab-4f5d-a603-ee68d7f6b285 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:57Z" name: model-chainer-64698579b9-p8c24.18c683f1f2d2a709 namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 647e72c7-c6a2-419f-bb39-aab90edb6520 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T17:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:57Z" name: model-chainer-64698579b9-p8c24.18c683f1f620b582 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 78fe877c-6383-4bc0-9d2a-5d8e3e6c6fb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T17:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:57Z" name: model-chainer-64698579b9-p8c24.18c683f1f6917cb6 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: faa98538-a5a5-43ca-9f19-0495d36ea7ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:06Z" name: model-chainer-64698579b9-p8c24.18c683f40eb70418 namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: d7f8d4af-9d55-487c-95f1-868441814a37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-p8c24 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: cca1b114-14e7-4e18-af5e-a9b065003977 kind: Event lastTimestamp: "2026-07-28T17:23:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:34Z" name: model-chainer-64698579b9-p8c24.18c683f4a2a125aa namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 6c58b8c0-8b1f-481a-96a6-c08aa8bd1671 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-64698579b9 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 911a7075-9661-4478-94d5-ca55847881d8 kind: Event lastTimestamp: "2026-07-28T17:22:56Z" message: 'Created pod: model-chainer-64698579b9-p8c24' metadata: creationTimestamp: "2026-07-28T17:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:56Z" name: model-chainer-64698579b9.18c683f1b3a7e61d namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: f4018c64-8a96-4a25-aa9a-a902f85bc363 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: d4005b32-de5d-41a8-855d-a4a1dab2a1dd kind: Event lastTimestamp: "2026-07-28T17:22:56Z" message: Scaled up replica set model-chainer-64698579b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:56Z" name: model-chainer.18c683f1b2ba2bdd namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 47259c1e-b346-4ff5-a574-1c040fa2c288 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 327ccb54-14c7-4650-ac18-c0924351f678 kind: Event lastTimestamp: "2026-07-28T17:22:56Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T17:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:56Z" name: model-chainer.18c683f1b687735f namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 519241f5-9474-4c1e-b3c7-03cc4acda730 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:31.644467Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1bf99-7658f8b77-btrts to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:31Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c6848599c7507a namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 050342f9-22e8-41a8-ade6-a9ad73560417 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:32Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c68485b3b28c10 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: c5d5dcfe-e5a8-4301-9f19-f8473cd7fe31 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1bf99} kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c68485b4f0811d namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: dc299b46-cae8-456b-9988-2906c75e096c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1bf99} kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: 'Created container: sequence-graph-1bf99' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c68485b81f895b namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: dc64ca66-9aa6-4d36-844b-4ab08d69462a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1bf99} kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Started container sequence-graph-1bf99 metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c68485b89455f5 namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 8916ead2-31e6-422b-8adc-63910843273d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1bf99} kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Stopping container sequence-graph-1bf99 metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:45Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c68488dfeb7f03 namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: e8419c19-5d99-4ac3-a15c-c586bca4a1b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1bf99} kind: Pod name: sequence-graph-1bf99-7658f8b77-btrts namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 9e66f4a8-d18c-4aa9-bd03-8511b81f2c26 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: sequence-graph-1bf99-7658f8b77-btrts.18c684899438c786 namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: 787cdecc-9466-4fa7-b75b-bf9fac393599 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1bf99-7658f8b77 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 55c48962-d081-4cd4-af07-9f7d1c453062 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Created pod: sequence-graph-1bf99-7658f8b77-btrts' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:31Z" name: sequence-graph-1bf99-7658f8b77.18c68485995af8c2 namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: f7894bec-d7a1-4fc7-9ada-591764238609 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 7627fdf8-53ae-410e-bfbc-bc4a70d70b84 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Scaled up replica set sequence-graph-1bf99-7658f8b77 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:31Z" name: sequence-graph-1bf99.18c68485985504da namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: f9bd7275-ad76-4fdf-979d-87ea603b76e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: 2ab236ca-cb73-4b19-9b56-40f423386545 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: InferenceGraph [sequence-graph-1bf99] is Ready metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:31Z" name: sequence-graph-1bf99.18c684859c666c15 namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 8224ea7c-8929-42b9-a486-5a93fd98a26d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:24:06.799168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3a5ae-6966f7f449-bgxjl to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:24:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:24:06Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c6840216601a62 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: d87f06c4-1cc4-4e9c-b707-6615267ffbf7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-3a5ae-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:06Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c6840221da0012 namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: b3a5ff4f-9161-48e0-b2fb-02d386d0c207 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:07Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:24:07Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:24:07Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c6840253b0a524 namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: 680e6e19-173f-4a8b-a6ff-a78f944f29f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a5ae} kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:24:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:07Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c6840255bed365 namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: 036c1d23-143c-4428-8e21-7553edc1a205 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a5ae} kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:24:07Z" message: 'Created container: sequence-graph-3a5ae' metadata: creationTimestamp: "2026-07-28T17:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:07Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c6840258e58104 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: 8119384d-1907-4b61-8351-89d23d04738f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a5ae} kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:24:07Z" message: Started container sequence-graph-3a5ae metadata: creationTimestamp: "2026-07-28T17:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:07Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c68402595fb7ff namespace: kserve-ci-e2e-test resourceVersion: "17085" uid: f0eaf3e0-62a9-4dfb-b0f6-4245673dfb3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a5ae} kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Stopping container sequence-graph-3a5ae metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:21Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c684753c77d8ca namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: dd17921c-33c4-4a53-8ac4-fed316730080 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a5ae} kind: Pod name: sequence-graph-3a5ae-6966f7f449-bgxjl namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: acd97157-d6c7-49b7-b431-5b9be226c95a kind: Event lastTimestamp: "2026-07-28T17:32:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:49Z" name: sequence-graph-3a5ae-6966f7f449-bgxjl.18c68475ed627915 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 5e65cc7f-065b-44c0-b1e3-7cc6eabfdc3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3a5ae-6966f7f449 namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: 40b2bfab-7521-40c9-8efc-0576880f2f9a kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: 'Created pod: sequence-graph-3a5ae-6966f7f449-bgxjl' metadata: creationTimestamp: "2026-07-28T17:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:06Z" name: sequence-graph-3a5ae-6966f7f449.18c6840215416cfd namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: 40b36595-d571-4305-8a97-d495b0030927 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 7716d205-44ee-4f3c-909c-e901c768ace6 kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: Scaled up replica set sequence-graph-3a5ae-6966f7f449 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:06Z" name: sequence-graph-3a5ae.18c6840213bb9d6e namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 555682b8-b466-4647-aadd-162a51db441c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: ec6be8e3-40cd-4d16-adb2-0dfcdd9324b4 kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: InferenceGraph [sequence-graph-3a5ae] is Ready metadata: creationTimestamp: "2026-07-28T17:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:24:06Z" name: sequence-graph-3a5ae.18c684021e1aa1d0 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 5e87f446-fff8-4043-94ca-56f532652a45 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:24:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: f48bf3bb-fd70-49db-84f4-7f7a9ad30888 kind: Event lastTimestamp: "2026-07-28T17:24:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:36Z" name: sequence-graph-3a5ae.18c6840592d2e8a9 namespace: kserve-ci-e2e-test resourceVersion: "17294" uid: 866ee15b-5cd3-4957-9b43-cc61c87285ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:24:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: f48bf3bb-fd70-49db-84f4-7f7a9ad30888 kind: Event lastTimestamp: "2026-07-28T17:24:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:36Z" name: sequence-graph-3a5ae.18c6840593360d0b namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 130666d5-8d88-44da-bc29-0d1478c38b40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:45.964123Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22413" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9d7bb-645c799655-ks4zp to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:45Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c68496e793645a namespace: kserve-ci-e2e-test resourceVersion: "22419" uid: dc7218a6-89c9-4397-91d5-ca97c4d3547a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-9d7bb-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:46Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c68496eed14d4c namespace: kserve-ci-e2e-test resourceVersion: "22433" uid: 4081ec40-7d29-4cbe-a8d5-9c5f86cf7deb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:46Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c6849724d1a27b namespace: kserve-ci-e2e-test resourceVersion: "22440" uid: c5601d75-e0f9-4b0b-a82c-9160188c2f04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9d7bb} kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:34:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:47Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c68497261e4f7a namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: 4b92db13-8f53-44fd-8491-cb13c102f2fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9d7bb} kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:34:47Z" message: 'Created container: sequence-graph-9d7bb' metadata: creationTimestamp: "2026-07-28T17:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:47Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c6849729cc0bf6 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: bdbab5a7-0ecd-4bf5-8fa9-22037010b632 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9d7bb} kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:34:47Z" message: Started container sequence-graph-9d7bb metadata: creationTimestamp: "2026-07-28T17:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:47Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c684972a42a3dc namespace: kserve-ci-e2e-test resourceVersion: "22444" uid: 7c2d5270-2314-4952-896c-3e0bc19d4ec6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9d7bb} kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Stopping container sequence-graph-9d7bb metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:00Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c6850a15ccde22 namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 627484d6-10f0-423e-bec5-255ab503d797 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9d7bb} kind: Pod name: sequence-graph-9d7bb-645c799655-ks4zp namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 39de043e-47ed-4211-93c5-1c76105cd383 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: sequence-graph-9d7bb-645c799655-ks4zp.18c6850ae0b6db0b namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: 7f488a07-911b-4d63-bee1-54f3de05f8df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9d7bb-645c799655 namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: 54f58591-0015-4b10-b05e-ef62132a4cb0 kind: Event lastTimestamp: "2026-07-28T17:34:45Z" message: 'Created pod: sequence-graph-9d7bb-645c799655-ks4zp' metadata: creationTimestamp: "2026-07-28T17:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:45Z" name: sequence-graph-9d7bb-645c799655.18c68496e723b21e namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: f5d5a177-f7ca-49d6-b24f-9ad41006bd5c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: d10074b1-7cd6-4453-b343-f8c61f4fadeb kind: Event lastTimestamp: "2026-07-28T17:34:45Z" message: Scaled up replica set sequence-graph-9d7bb-645c799655 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:45Z" name: sequence-graph-9d7bb.18c68496e639c766 namespace: kserve-ci-e2e-test resourceVersion: "22411" uid: 72754f24-0b54-414b-9fbc-90b5b04287bc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "22425" uid: ec8d5140-1dcc-4487-8b74-049f0c8fdf69 kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: InferenceGraph [sequence-graph-9d7bb] is Ready metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:46Z" name: sequence-graph-9d7bb.18c68496ea060578 namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: e68949e4-5571-4ef2-a794-e6ec6b44232b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: c82be2c0-c247-416a-937c-665ea9f91314 kind: Event lastTimestamp: "2026-07-28T17:35:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:30Z" name: sequence-graph-9d7bb.18c6849a64930c23 namespace: kserve-ci-e2e-test resourceVersion: "22768" uid: 0dacd95c-4280-405c-b607-ba5047866c9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: c82be2c0-c247-416a-937c-665ea9f91314 kind: Event lastTimestamp: "2026-07-28T17:35:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:30Z" name: sequence-graph-9d7bb.18c6849a64f5885b namespace: kserve-ci-e2e-test resourceVersion: "22769" uid: 5371fd6e-c341-40db-8494-233018852e6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:01.514675Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4f50a-747967f6f4-dwptv to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:01Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c6851840f7ed22 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: ae76dd6a-a04a-4a5a-aa00-017c948d480f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:01Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:44:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-4f50a-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:01Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c685184ee75609 namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: 6e102251-4801-47c4-9ee9-2c0069c68a24 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:44:02Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:02Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c685187e7f1b45 namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: f8e45612-dfd2-4648-8d21-c99c4debb29a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f50a} kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:44:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:02Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c685187fec9a13 namespace: kserve-ci-e2e-test resourceVersion: "27257" uid: e127a484-270a-4573-ae6f-5c95ea275e45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f50a} kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:44:02Z" message: 'Created container: splitter-graph-4f50a' metadata: creationTimestamp: "2026-07-28T17:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:02Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c68518837f4bdc namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: b0dc58cc-cab5-4303-9180-82bf6534e632 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f50a} kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:44:02Z" message: Started container splitter-graph-4f50a metadata: creationTimestamp: "2026-07-28T17:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:02Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c6851883f2992e namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: b00ccb4e-5042-42d8-92c7-8c037da744b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f50a} kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:52:16Z" message: Stopping container splitter-graph-4f50a metadata: creationTimestamp: "2026-07-28T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:52:16Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c6858b6dffaf25 namespace: kserve-ci-e2e-test resourceVersion: "30758" uid: 7aa508df-17c2-403d-8eec-6cc2ab7a111a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:52:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f50a} kind: Pod name: splitter-graph-4f50a-747967f6f4-dwptv namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 11ad9f23-e3f4-4d26-b401-37dab78becc3 kind: Event lastTimestamp: "2026-07-28T17:52:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:52:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:52:44Z" name: splitter-graph-4f50a-747967f6f4-dwptv.18c6858c2fdfaa70 namespace: kserve-ci-e2e-test resourceVersion: "30993" uid: daf07565-0438-474b-9803-2bc732107ea1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4f50a-747967f6f4 namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: 20cbd138-6b6c-4d11-b54b-b5be570a6bef kind: Event lastTimestamp: "2026-07-28T17:44:01Z" message: 'Created pod: splitter-graph-4f50a-747967f6f4-dwptv' metadata: creationTimestamp: "2026-07-28T17:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:01Z" name: splitter-graph-4f50a-747967f6f4.18c68518408838d9 namespace: kserve-ci-e2e-test resourceVersion: "27226" uid: fe6cbd89-fa0b-4afc-955a-9bf471639e2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 6dbff08f-97a2-476d-bfe1-7caa6c2f788d kind: Event lastTimestamp: "2026-07-28T17:44:01Z" message: Scaled up replica set splitter-graph-4f50a-747967f6f4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:01Z" name: splitter-graph-4f50a.18c685183f9f798d namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: 89e2fefc-132a-4a24-ad08-a34ccaa64f79 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27237" uid: 0eca9a58-3eb0-4f8a-92b4-63c9431e484f kind: Event lastTimestamp: "2026-07-28T17:44:01Z" message: InferenceGraph [splitter-graph-4f50a] is Ready metadata: creationTimestamp: "2026-07-28T17:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:01Z" name: splitter-graph-4f50a.18c68518459ac6b5 namespace: kserve-ci-e2e-test resourceVersion: "27240" uid: 9c4c7877-1b86-42a7-8751-90d147365074 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27220" uid: 1106b5b4-85b1-44e5-bb65-90da9eaa4519 kind: Event lastTimestamp: "2026-07-28T17:44:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:16Z" name: splitter-graph-4f50a.18c6851bbe11c878 namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: 94830ff1-c5a2-4692-b7a7-be3f575af9a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27220" uid: 1106b5b4-85b1-44e5-bb65-90da9eaa4519 kind: Event lastTimestamp: "2026-07-28T17:44:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:16Z" name: splitter-graph-4f50a.18c6851bbe80f264 namespace: kserve-ci-e2e-test resourceVersion: "27402" uid: 5e3f615d-ad12-4e8f-abb0-d2c7c7e8b68f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27401" uid: 1106b5b4-85b1-44e5-bb65-90da9eaa4519 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:31Z" name: splitter-graph-4f50a.18c6851f3d527776 namespace: kserve-ci-e2e-test resourceVersion: "27521" uid: 6992feb7-8416-473d-b9cb-d4da020a9a4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27401" uid: 1106b5b4-85b1-44e5-bb65-90da9eaa4519 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:31Z" name: splitter-graph-4f50a.18c6851f3dac8fc5 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: 55ba6906-ac3e-47eb-adab-832f0cff8b57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:42:51.179100Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b5d39-69cfbcb448-srpzh to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c68507e0a4495f namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: bd68092e-d2d1-4499-9ca6-fe6a78d72908 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c68507fa419101 namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: c839fa4b-0af2-4905-9024-de6f3d3ac5de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5d39} kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c68507fbc6ba7a namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: 0266e81d-cf79-4314-9b8e-99c8020fba63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5d39} kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: 'Created container: splitter-graph-b5d39' metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c68507fe827053 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 8c05f5ec-5c56-4772-b822-3b0e250bec85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5d39} kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: Started container splitter-graph-b5d39 metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c68507fef70a08 namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 0e86efe0-87d4-4b63-8388-2cba999d0261 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5d39} kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Stopping container splitter-graph-b5d39 metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c6850a38a41edc namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: e942446a-a264-4616-afd5-4197a08789b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5d39} kind: Pod name: splitter-graph-b5d39-69cfbcb448-srpzh namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 5ced0663-42d5-480d-ba09-ced7de0f4d58 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: splitter-graph-b5d39-69cfbcb448-srpzh.18c6850aa8b2fe8a namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 6a539a4f-e8e7-4a27-a111-8aee7708ef18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b5d39-69cfbcb448 namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: 2a7af777-7246-4b26-a9aa-5138fc5ad1eb kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: 'Created pod: splitter-graph-b5d39-69cfbcb448-srpzh' metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39-69cfbcb448.18c68507e0274b79 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: b50451d9-a346-4960-8e99-07c93d3e3642 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: 0bae3070-d668-43de-95f4-05647dd37a49 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: Scaled up replica set splitter-graph-b5d39-69cfbcb448 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39.18c68507def695f9 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: dd4d6ea9-7b93-4d5e-81c8-4616823be5e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 57856055-6d2d-4f77-a8d9-ca7a7f56ac52 kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: InferenceGraph [splitter-graph-b5d39] is Ready metadata: creationTimestamp: "2026-07-28T17:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:51Z" name: splitter-graph-b5d39.18c68507e37d6b98 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 77225a91-fbf0-45f1-aea3-94d14fb42b3d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:21.596729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c684754a9bb133 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b026b11b-2b42-423f-8047-0129c4ddff84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1bf99-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c68475557ab162 namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 19e51c8a-718b-4d30-bc68-7a1f7dddc433 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c68475880e0ca3 namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: e24bd7a9-eefc-4dd0-be0b-f86b915a67ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c6847589820191 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 1cd26c16-b92d-4281-8d87-e613d820ea38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c684758cc3a26a namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: f983d2fe-6e20-4352-8dba-5b713d870a3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c684758d31ff91 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: a35bcb9b-1fae-430e-b816-b7952d0640e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c684758d3b7c72 namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: 9a6bb5d1-2a6d-4319-ba4a-0f1208fd2275 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c684759078a2b3 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: ed1b5d7e-9761-44c0-b374-d80cba54161b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:32:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:22Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c6847590e9242c namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: 7e88a31d-6a45-491e-9f2c-0e9ab15587e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:33:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:09Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c68475a9b047fe namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 8f938e9f-8eb6-4fae-adae-d961b173bf7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:45Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c68488e6f10b51 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: cac0d0ee-fba8-4c46-8069-5be9f713b9ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: cba979ea-2b42-440f-8b13-493dff0c9bbd kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:45Z" name: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4.18c68488e6f1f885 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 4fe66423-7bf5-4360-b08c-768a44e9e263 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1bf99-predictor-7c44b59b85 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 1deefe7a-8a4b-4d21-89d0-476ab194052a kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'Created pod: success-200-isvc-1bf99-predictor-7c44b59b85-8kcd4' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-1bf99-predictor-7c44b59b85.18c684754a2f2bc3 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: fc38fd26-d495-4078-ac44-ee61ab3c72a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 286ba375-6f16-42e1-b64b-10aec9cea0b8 kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Scaled up replica set success-200-isvc-1bf99-predictor-7c44b59b85 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-1bf99-predictor.18c68475486c5e0d namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 4cba7ed1-15eb-4989-a471-5a9acab9b92b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 27aa75f2-35ec-4f19-8cc5-08ddd2d79c75 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: success-200-isvc-1bf99-predictor.18c68478c6f903ef namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 2546996c-ebaa-4c3f-b67a-646a892505f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf99-predictor namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 27aa75f2-35ec-4f19-8cc5-08ddd2d79c75 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: success-200-isvc-1bf99-predictor.18c68478c75ae702 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 25e2007c-a073-4632-8fd6-76351f2bef78 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 94ff9d8a-6a8d-4b0a-8f7b-40ae6a0d3a84 kind: Event lastTimestamp: "2026-07-28T17:33:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:19Z" name: success-200-isvc-1bf99.18c6847548a07c2d namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 907c9d53-6ac7-4751-add1-594ee0f6f08d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 94ff9d8a-6a8d-4b0a-8f7b-40ae6a0d3a84 kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-1bf99": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1bf99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-1bf99.18c6847549095e29 namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: 3c4f3c00-0af3-4abd-9bf0-6c6db7d19523 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 94ff9d8a-6a8d-4b0a-8f7b-40ae6a0d3a84 kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1bf99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-1bf99.18c684754909b37a namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: 3ea34147-6733-4525-a921-5d91c7e2dc63 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf99 namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 94ff9d8a-6a8d-4b0a-8f7b-40ae6a0d3a84 kind: Event lastTimestamp: "2026-07-28T17:33:19Z" message: InferenceService [success-200-isvc-1bf99] is Ready metadata: creationTimestamp: "2026-07-28T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:19Z" name: success-200-isvc-1bf99.18c68482b9568303 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: cfde88d2-b67e-4bf4-9a48-3b141abc6be5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:22:10.858339Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e717c3752a namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: f1d1351f-599f-4342-b272-6897b40097a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e731860b9b namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e97e49c1-9a62-40f8-b825-41d7da63cec2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e732f7edb4 namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: e2ed3c79-6a33-46ea-b576-b456f7c35e14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e7373fa0f4 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: f78fb1ed-61ed-4c65-8d50-cc8fe954366c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e737b0fb03 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: e84ca328-4b6c-4a2f-bae6-654a12b63efe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e737ba08ec namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 953fab0f-d5c8-44fa-b086-ab16dee598e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e73b401894 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 3f7ea1c7-f1c4-4103-a3a9-a6dc7bde4c98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:22:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:11Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e73be67cb2 namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: f3136a34-5a35-4a80-864e-f008ee888294 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:31:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:27Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c683e74b822f62 namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: da433f01-6ed6-4f1a-aa46-2a4f9601632c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c684684b21f401 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 6b5e3f3e-1faa-45b5-9479-799196007808 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c684684b22a809 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: aefef42e-1882-427e-ad8c-ee268dfd73ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 73c17a24-db85-4298-a8f4-762c1077759e kind: Event lastTimestamp: "2026-07-28T17:31:27Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:27Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x.18c68468c027e554 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: 90b0a44f-cc64-4f65-9603-bfcd0b29d96f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1fbfc-predictor-6c545c88dc namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 4a6cd1e7-7959-4833-b0c5-145ce655a50c kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: 'Created pod: success-200-isvc-1fbfc-predictor-6c545c88dc-pt88x' metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-1fbfc-predictor-6c545c88dc.18c683e7178cba37 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 091d0cbe-47c4-434b-97f4-0bea16cda2bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: d2106041-1993-4624-82af-8e94c8133937 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Scaled up replica set success-200-isvc-1fbfc-predictor-6c545c88dc from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-1fbfc-predictor.18c683e714419f68 namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 1aa0ab5b-478d-469a-a314-f8f9741a52a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:22:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 441ed370-99d2-4c6e-98e2-bd546bf13bf5 kind: Event lastTimestamp: "2026-07-28T17:22:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:40Z" name: success-200-isvc-1fbfc-predictor.18c683ea92ad2a68 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: ce13e14f-4c45-4071-8f15-8a13fbd9ecdf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:22:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 441ed370-99d2-4c6e-98e2-bd546bf13bf5 kind: Event lastTimestamp: "2026-07-28T17:22:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:40Z" name: success-200-isvc-1fbfc-predictor.18c683ea93165556 namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: 7a8fdc5a-917b-470d-a441-4eb7f13770db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 441ed370-99d2-4c6e-98e2-bd546bf13bf5 kind: Event lastTimestamp: "2026-07-28T17:23:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:25Z" name: success-200-isvc-1fbfc-predictor.18c683f19079412d namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: 123c5c68-3d76-4258-89b9-40cbbdff45fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fbfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 441ed370-99d2-4c6e-98e2-bd546bf13bf5 kind: Event lastTimestamp: "2026-07-28T17:23:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:25Z" name: success-200-isvc-1fbfc-predictor.18c683f191022854 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 8fc7f651-b9c2-413c-8223-5a3d012ddb46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 7cab7158-62f9-4a46-831a-c911ea2015ef kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:57Z" name: success-200-isvc-1fbfc.18c683e7145705f2 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: cd388c58-3d24-4a05-82c9-bfc1ce00a6c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 7cab7158-62f9-4a46-831a-c911ea2015ef kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-1fbfc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1fbfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-1fbfc.18c683e714d85ca3 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 053bdd10-1d05-42f4-bfd4-609f454fd69d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 7cab7158-62f9-4a46-831a-c911ea2015ef kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1fbfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-1fbfc.18c683e714d8cfd6 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: b865c350-4953-4708-8ba7-64024c38234f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 7cab7158-62f9-4a46-831a-c911ea2015ef kind: Event lastTimestamp: "2026-07-28T17:22:57Z" message: InferenceService [success-200-isvc-1fbfc] is Ready metadata: creationTimestamp: "2026-07-28T17:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:57Z" name: success-200-isvc-1fbfc.18c683f20a8ef153 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: b046b891-e8eb-4e7a-9cf4-9d619651563e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:23:06.735305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3a5ae-predictor-69d6694669-t9t26 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:23:06Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f41a4a4194 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: b60497d9-d322-419a-9282-6494a199ecfc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f43414ee39 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 2718bb5c-5bf8-4a28-8ff0-86fad57c6229 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f43565da9b namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: cd387081-5ac4-4cdf-9a35-64690ba22718 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f43827b476 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 8218d5e8-09b1-4c19-a73e-c004be7beb98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f43894bb43 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e22f5910-1504-41de-8f03-475996745af0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f4389efd17 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: ed5dcb6f-a955-400c-ba17-ff72b79d8cf4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f43bc3369e namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 07181e17-5cb1-42ab-bf05-62001c8ed2a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:07Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f43c33fb0a namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 4b91477b-3f73-4e07-9ec9-157643421e64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:23:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:44Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c683f4640db847 namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: be2abbf6-c65b-46d8-baef-061ca0651f7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c68475454e8c88 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 0d8d093b-c4bb-44ec-959e-f1a3a10bb3f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: ffd61427-c7ab-4d72-aa02-a8018c09322b kind: Event lastTimestamp: "2026-07-28T17:32:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:21Z" name: success-200-isvc-3a5ae-predictor-69d6694669-t9t26.18c6847545533145 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: d8ff4d13-8e10-4130-89a9-947c8d50660b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3a5ae-predictor-69d6694669 namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: ef9b5c91-0843-4bfa-99eb-3db253cfe67d kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: 'Created pod: success-200-isvc-3a5ae-predictor-69d6694669-t9t26' metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:06Z" name: success-200-isvc-3a5ae-predictor-69d6694669.18c683f419be38b9 namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 3b4f60c9-9734-44df-ae2f-16a9e27da152 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: f2b4c913-6915-40f9-b2db-dec5f6da4f83 kind: Event lastTimestamp: "2026-07-28T17:23:06Z" message: Scaled up replica set success-200-isvc-3a5ae-predictor-69d6694669 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:06Z" name: success-200-isvc-3a5ae-predictor.18c683f41815e30b namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 8be16e8c-6ce5-48bd-b8ed-d213ce92dae2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: 96218b1e-9070-4edf-99cd-045ca5a0aadc kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:06Z" name: success-200-isvc-3a5ae-predictor.18c683f79718b646 namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: 12d61ec6-b53b-48c6-af79-c6561565f85b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a5ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: 96218b1e-9070-4edf-99cd-045ca5a0aadc kind: Event lastTimestamp: "2026-07-28T17:24:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:06Z" name: success-200-isvc-3a5ae-predictor.18c683f7976a324f namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: d8c395d9-cbdb-477a-a93b-29ef7ea346db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:23:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: e54df3aa-0976-49b4-a8ca-d772c167d5e5 kind: Event lastTimestamp: "2026-07-28T17:23:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:55Z" name: success-200-isvc-3a5ae.18c683f4182156a7 namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: b5fccd05-8604-44f4-beca-7576ffa9c890 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a5ae namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: e54df3aa-0976-49b4-a8ca-d772c167d5e5 kind: Event lastTimestamp: "2026-07-28T17:23:55Z" message: InferenceService [success-200-isvc-3a5ae] is Ready metadata: creationTimestamp: "2026-07-28T17:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:55Z" name: success-200-isvc-3a5ae.18c683ff5ba9ba57 namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: c3fcaf7f-00db-4931-9d02-915ec3d3d77a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:01.503177Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4f50a-predictor-569ccbb779-gwzcj to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a48012e39 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: c8a59a67-e08e-4e89-b444-6b79d8c7230a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4f50a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a54766214 namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: f56386f1-8a1d-4837-83d5-9f8e4d2f1b79 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a85804862 namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 86fcbb31-6455-4527-bac9-c5182232ed70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a8717a137 namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f229b17c-0e4d-426d-998d-17932ffd724c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a89b0ef7c namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: a835866c-dd2c-46a8-bab7-43d4900c725a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a8a1dbfe4 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: ff027caf-0502-4674-aee0-0efb04f114a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a8a25fb4b namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 35a4116b-7cbb-4f32-a250-c061bc07aa0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a8de7ba50 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: b2671a0a-ee52-43b0-84e2-95adcc071480 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:02Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850a8e54d7a0 namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: 9862d413-d9ca-4cdf-b686-7ccbd7761143 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:43:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:39Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6850ad317d49a namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: a719dabc-77ff-411e-b83d-d8f82ef0b100 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:52:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:52:16Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6858b76a5d29e namespace: kserve-ci-e2e-test resourceVersion: "30773" uid: 28e7f873-246f-48ce-8e20-365ea61aa783 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: ea7b2898-7d16-4d1f-85ab-9c12e6e8508f kind: Event lastTimestamp: "2026-07-28T17:52:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:52:16Z" name: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj.18c6858b76a63c08 namespace: kserve-ci-e2e-test resourceVersion: "30774" uid: 39d93165-f433-4998-a9fa-85ab99e32d09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4f50a-predictor-569ccbb779 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: e2d054e7-be3d-4128-95c9-4a93e5d38b9a kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: 'Created pod: success-200-isvc-4f50a-predictor-569ccbb779-gwzcj' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-4f50a-predictor-569ccbb779.18c6850a4768341d namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 82b24b36-e9fe-4516-aff9-e77bdf68d5a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: 6f9b3a39-13ce-4384-9766-50fc7ea7c2cf kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Scaled up replica set success-200-isvc-4f50a-predictor-569ccbb779 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-4f50a-predictor.18c6850a4578cf18 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: b9b45f71-3a8b-47e8-aa49-5a598e1ef9bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 066e0905-bcbe-47de-b098-e947b25aee8a kind: Event lastTimestamp: "2026-07-28T17:43:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:31Z" name: success-200-isvc-4f50a-predictor.18c6850dc3f7f185 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 3eb0510d-4f41-4371-83cb-34a7e133af57 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 066e0905-bcbe-47de-b098-e947b25aee8a kind: Event lastTimestamp: "2026-07-28T17:43:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:31Z" name: success-200-isvc-4f50a-predictor.18c6850dc468c230 namespace: kserve-ci-e2e-test resourceVersion: "26946" uid: 05464d59-60a0-4657-90a8-d76399380470 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: 066e0905-bcbe-47de-b098-e947b25aee8a kind: Event lastTimestamp: "2026-07-28T17:44:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:16Z" name: success-200-isvc-4f50a-predictor.18c68514c1d60e12 namespace: kserve-ci-e2e-test resourceVersion: "27403" uid: ca7a430b-179d-4d18-a41b-6699d89146d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f50a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: 066e0905-bcbe-47de-b098-e947b25aee8a kind: Event lastTimestamp: "2026-07-28T17:44:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:16Z" name: success-200-isvc-4f50a-predictor.18c68514c250a81d namespace: kserve-ci-e2e-test resourceVersion: "27404" uid: ae7c2e62-01b1-48f7-b0c1-2a90f8cf0004 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4f50a namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: 078c85ef-fe76-4ceb-af31-ae47e99c6fd4 kind: Event lastTimestamp: "2026-07-28T17:43:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:50Z" name: success-200-isvc-4f50a.18c6850a457f4e6f namespace: kserve-ci-e2e-test resourceVersion: "27112" uid: 64c85029-7c20-430a-8318-ec44e724aa4a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4f50a namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: 078c85ef-fe76-4ceb-af31-ae47e99c6fd4 kind: Event lastTimestamp: "2026-07-28T17:43:49Z" message: InferenceService [success-200-isvc-4f50a] is Ready metadata: creationTimestamp: "2026-07-28T17:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:49Z" name: success-200-isvc-4f50a.18c68515927d05fc namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: 5003ca56-8afe-4b1d-a90e-51d872fb57fb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:45.895151Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:45Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c68488eb2f7e51 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 49207115-304f-43a3-95a2-59074684be6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9d7bb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c68488f51ba084 namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 4d445efe-d08c-436e-a505-93b83f738a7f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c684892890d437 namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 99f58651-8de8-4805-86d0-e489d613c3c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c684892a0081e1 namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: d616c478-1e5d-4466-a2c1-95de0dc104b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c684892ce9b49c namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: cf8a51bb-fe63-44ca-b7af-6bbbe71d56df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:47Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c684892d573741 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: ae6036c4-4fd0-4ab4-ad52-588d996a59b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:47Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c684892d613d8d namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 72cbf5a7-0a56-4b64-9bf5-74355b5f6fee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:47Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c68489307ed89c namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: e8d9a60a-9268-4eac-97e3-49020dcd00d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:33:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:47Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c6848930f1df5d namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 10300ab5-3e37-47ba-bff3-76c8304e7e4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:33:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c6848950d21a66 namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: 46094ba2-3f18-4d29-aba9-7cfa9c30b2dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c6850a1d9343b9 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: 4600e534-3167-4a2a-9480-41653df5ab8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c6850a1d93b183 namespace: kserve-ci-e2e-test resourceVersion: "26416" uid: 060d1c2a-19c4-4e77-abfb-33397d1feeff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 8174e7bd-0b17-4167-a5f0-28e6398e6f74 kind: Event lastTimestamp: "2026-07-28T17:43:03Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:03Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt.18c6850ac54cd262 namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 14de0414-31f5-40af-863d-97ccea30a20c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 42858c72-7343-4761-81b9-358ac6c34a36 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: 'Created pod: success-200-isvc-9d7bb-predictor-7c9b6cdf9f-445tt' metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:45Z" name: success-200-isvc-9d7bb-predictor-7c9b6cdf9f.18c68488eac5a6b5 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: 8ec28de9-a5ac-45a8-902e-708d530f0aa0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 4e089f9b-9581-450e-81e1-27a3ab8af067 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: Scaled up replica set success-200-isvc-9d7bb-predictor-7c9b6cdf9f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:45Z" name: success-200-isvc-9d7bb-predictor.18c68488e8f0a2f0 namespace: kserve-ci-e2e-test resourceVersion: "21836" uid: b59480a9-7508-4ff3-b61b-ae17e1bcf4e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 5d3201de-4579-4c67-aafd-99aad3b22ee2 kind: Event lastTimestamp: "2026-07-28T17:34:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:00Z" name: success-200-isvc-9d7bb-predictor.18c6848c67623094 namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: 7db63888-4e95-4e54-b992-773a445d10cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 5d3201de-4579-4c67-aafd-99aad3b22ee2 kind: Event lastTimestamp: "2026-07-28T17:34:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:00Z" name: success-200-isvc-9d7bb-predictor.18c6848c67c7f2fc namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: 08173c78-5f99-4b23-9985-2f96b6324cbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 5d3201de-4579-4c67-aafd-99aad3b22ee2 kind: Event lastTimestamp: "2026-07-28T17:35:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:00Z" name: success-200-isvc-9d7bb-predictor.18c6848fe65db0f9 namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: ecab3592-e72e-4239-a027-dc4cbfbd3770 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d7bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 5d3201de-4579-4c67-aafd-99aad3b22ee2 kind: Event lastTimestamp: "2026-07-28T17:35:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:00Z" name: success-200-isvc-9d7bb-predictor.18c6848fe6cef81e namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: f4d276cd-52d7-44d3-ba4d-5f2631efa3de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: bd753128-fb74-4d2f-be4f-00bd6d3a99bf kind: Event lastTimestamp: "2026-07-28T17:34:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:33Z" name: success-200-isvc-9d7bb.18c68488e90993ba namespace: kserve-ci-e2e-test resourceVersion: "22319" uid: ae27cfc8-5fe1-4538-bdc9-3d304562a0c4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d7bb namespace: kserve-ci-e2e-test resourceVersion: "22316" uid: bd753128-fb74-4d2f-be4f-00bd6d3a99bf kind: Event lastTimestamp: "2026-07-28T17:34:33Z" message: InferenceService [success-200-isvc-9d7bb] is Ready metadata: creationTimestamp: "2026-07-28T17:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:33Z" name: success-200-isvc-9d7bb.18c684940f8f9fd6 namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: bcc933e2-02fe-4f47-94e7-36bcd9b4f3ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:31:25.902699Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20153" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c6846852fc3d93 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: d1d06c15-4002-4ee9-a50e-7256b5fac210 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c684686cfe0f1e namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 95ed8e54-87b1-46a8-a149-84329168558b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c684686e5b9a31 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 91b0de1d-42f3-4dff-bb77-376c0680db1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c6846871e5ea75 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 08f2f1d7-bd85-467e-a1d0-7a4ec1736971 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c684687264eedc namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e1210215-3e63-4e17-936f-ddff2f5d7dcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c68468726d3c72 namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 9da14611-04dc-4fdb-8a70-6f4fca147f41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c684687549b285 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: 714f7ec8-eea1-40b3-85df-c490899fe70f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:31:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:26Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c6846875bf59b5 namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 2ad2993e-6f4a-4b9e-8e9a-7883babf0f7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:32:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:03Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c68468ccb15919 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 5127ab95-46a5-40da-9e5c-ba4e4bb617b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:36Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c68478a920bdf2 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 48b24426-eca7-4ac1-99c7-71c4c6d7c1aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:36Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c68478a921fe21 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: a6310593-1450-488c-8a2e-d558327fe53c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 9f2a77b3-e9a3-4590-af3e-4f5477b72454 kind: Event lastTimestamp: "2026-07-28T17:32:38Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:38Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh.18c6847954fca7e4 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 056e06f9-751f-419a-b3f7-c88940b15e53 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b0164-predictor-55bc6f9cdf namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 518991e6-3ee4-4786-83e3-3c4b91acdb9a kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: 'Created pod: success-200-isvc-b0164-predictor-55bc6f9cdf-jknhh' metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-b0164-predictor-55bc6f9cdf.18c68468527ab58c namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 398db026-1602-4af2-b959-accba67d536e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: 0147684e-dbc5-4f54-8381-5e75b5515d61 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Scaled up replica set success-200-isvc-b0164-predictor-55bc6f9cdf from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-b0164-predictor.18c684684e3fe797 namespace: kserve-ci-e2e-test resourceVersion: "20140" uid: 55c7a4c2-02c5-425c-96d4-3a078f23827e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 76873c16-4f9e-4675-90e8-ee4cafd53169 kind: Event lastTimestamp: "2026-07-28T17:31:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:55Z" name: success-200-isvc-b0164-predictor.18c6846bcbf4e2bd namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: edc9bc9e-0385-4205-9013-7d975fd3fed1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 76873c16-4f9e-4675-90e8-ee4cafd53169 kind: Event lastTimestamp: "2026-07-28T17:31:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:31:55Z" name: success-200-isvc-b0164-predictor.18c6846bcc5ca896 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: b5154280-8c84-4d5d-a494-7eed14727a9c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 76873c16-4f9e-4675-90e8-ee4cafd53169 kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:25Z" name: success-200-isvc-b0164-predictor.18c68472c9d28eff namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: c28964ab-835b-4dba-adc5-fe6f1c3b31f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b0164-predictor namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 76873c16-4f9e-4675-90e8-ee4cafd53169 kind: Event lastTimestamp: "2026-07-28T17:32:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:25Z" name: success-200-isvc-b0164-predictor.18c68472ca3d870a namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 4d6b4511-fa95-4593-9757-4ac1a53fb39d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 11191d21-731d-4599-a859-5fdaea3b70b1 kind: Event lastTimestamp: "2026-07-28T17:32:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:14Z" name: success-200-isvc-b0164.18c684684e3e4132 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 71e1cfaf-8573-4ecd-8cbd-e298a8661acf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 11191d21-731d-4599-a859-5fdaea3b70b1 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-b0164": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b0164": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-b0164.18c684684f0186e1 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 0f6f6e13-1689-4f02-bee1-1e0b066fd855 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 11191d21-731d-4599-a859-5fdaea3b70b1 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b0164": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:31:25Z" name: success-200-isvc-b0164.18c684684f01d0b9 namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 4ddd9fa3-2460-4e76-a157-844b7b87f496 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b0164 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 11191d21-731d-4599-a859-5fdaea3b70b1 kind: Event lastTimestamp: "2026-07-28T17:32:14Z" message: InferenceService [success-200-isvc-b0164] is Ready metadata: creationTimestamp: "2026-07-28T17:32:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:14Z" name: success-200-isvc-b0164.18c684738849aaa7 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: c5d01b25-1056-4848-99ab-5540d4d1fd7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:41:51.115307Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b5d39-predictor-6f5ff89745-nszhm to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684f9e4902b96 namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 37bfe9a1-c60b-4d4e-b032-ba3212e43b9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25731" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684f9fe38c920 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: 92ef68d3-a866-4ba7-afdd-3013e385f93b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684f9ffde00d4 namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: 525e4e17-60f5-4eaa-a1d4-10ff951c70ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684fa03f75f51 namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 6c272e1f-4675-4a51-a5d3-db9edd5b31dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684fa0474f374 namespace: kserve-ci-e2e-test resourceVersion: "25779" uid: 999e0421-b895-450e-88d8-4d37e5b8527c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684fa0480027d namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: 79184597-83f1-4041-96da-6a885a88390f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684fa0800be75 namespace: kserve-ci-e2e-test resourceVersion: "25783" uid: 52cee792-2d45-430e-9fbf-556effc82dbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684fa088619af namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 8f77a2e2-5470-43da-a152-4b6e6d19d550 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:41:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:42:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:29Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c684fa7405c91a namespace: kserve-ci-e2e-test resourceVersion: "26093" uid: 8d8bbe5b-6b24-4911-b8e6-dcf090b3eaca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c6850a4205b880 namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: 3627f239-3f35-48cb-a43f-4776b27e64f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c6850a42062112 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c390bd08-9708-42c4-bbcb-b44058a8906a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 5c5f73a8-86d6-4fb0-8a95-e4362eecd893 kind: Event lastTimestamp: "2026-07-28T17:43:04Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:04Z" name: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm.18c6850afc31c44a namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: e906476b-3abd-4727-81d4-0b0cf112730f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b5d39-predictor-6f5ff89745 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 7d4dedcf-dfc9-4388-82c0-f82fb52373be kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Created pod: success-200-isvc-b5d39-predictor-6f5ff89745-nszhm' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor-6f5ff89745.18c684f9e41e9ac2 namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: cf33299b-636a-42d1-be82-4ec2049c345e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25690" uid: c2c15f29-ae90-4883-ac65-8a2af6c517cb kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Scaled up replica set success-200-isvc-b5d39-predictor-6f5ff89745 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39-predictor.18c684f9e04cb6ee namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 06d778ee-82ea-4765-ae9b-cb61685d63a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:42:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: cbf11afb-62de-46f6-84eb-8fe90ec86a9a kind: Event lastTimestamp: "2026-07-28T17:42:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:21Z" name: success-200-isvc-b5d39-predictor.18c684fd5ef9f0fe namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 005b38fb-ac2f-4e27-8ff6-bbb4ad2103bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:42:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: cbf11afb-62de-46f6-84eb-8fe90ec86a9a kind: Event lastTimestamp: "2026-07-28T17:42:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:21Z" name: success-200-isvc-b5d39-predictor.18c684fd5f6830d9 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: 5d2ba392-ac00-4731-bd6f-c57291eed8f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:42:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: cbf11afb-62de-46f6-84eb-8fe90ec86a9a kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:42:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:51Z" name: success-200-isvc-b5d39-predictor.18c685045e09579e namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 39ecc124-6da8-4335-a732-d6801777e90a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:42:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5d39-predictor namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: cbf11afb-62de-46f6-84eb-8fe90ec86a9a kind: Event lastTimestamp: "2026-07-28T17:42:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:42:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:51Z" name: success-200-isvc-b5d39-predictor.18c685045e851dfd namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: 969d06b9-d264-4a56-9f3c-486d5724aebe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 1f63476f-af8b-436e-aef6-633b359fa7e9 kind: Event lastTimestamp: "2026-07-28T17:42:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:39Z" name: success-200-isvc-b5d39.18c684f9e077bf30 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 4d03f816-427d-43d0-b7e8-f89e201fc890 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 1f63476f-af8b-436e-aef6-633b359fa7e9 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-b5d39": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b5d39": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39.18c684f9e1244979 namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: bfde5c6b-f01c-4f20-8872-bf7e3c491c67 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 1f63476f-af8b-436e-aef6-633b359fa7e9 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b5d39": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-b5d39.18c684f9e1249411 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 2b168384-c2f9-4bf6-8489-041e99786111 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5d39 namespace: kserve-ci-e2e-test resourceVersion: "26190" uid: 1f63476f-af8b-436e-aef6-633b359fa7e9 kind: Event lastTimestamp: "2026-07-28T17:42:39Z" message: InferenceService [success-200-isvc-b5d39] is Ready metadata: creationTimestamp: "2026-07-28T17:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:39Z" name: success-200-isvc-b5d39.18c6850534f9918e namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 0a359fc7-b241-4322-b8fa-42390ee523a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:20:36.589532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:20:36Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d124e78b0b namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 473b49c1-1bcf-4f51-862b-fbb27a0b3b92 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:20:37Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d13eca35c8 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 098d56fc-e67b-40a6-bad4-b2c9353a3724 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:37Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:37Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d14077d3f2 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: c519e707-561e-426e-8f7d-61102a8d0160 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" in 17.526s (17.526s including waiting). Image size: 1334926941 bytes.' metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d5552810f6 namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: a6835c85-700c-4019-accd-f2905a4cee5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d559a790e7 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 9f879fe6-799b-4274-8f19-7a7c9786f5cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d55a21bfcc namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 5344af71-b501-41cb-ae84-29a3b8e38c6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:54Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d55a2fbd6b namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 2bd431a4-ac98-4356-97dc-4fc85e95c4ee reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.654s (2.654s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:57Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d5f870134f namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 2b0284d3-1d6a-498c-ae7c-59234a3ac9ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:57Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d5fb73bd73 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: 1ba36b77-e694-4a32-bc3c-f76b97e85e4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:20:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:20:57Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d5fbf4f884 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 0cec3549-a9b5-4427-b9d0-7a86d7123330 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:22:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:13Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683d5fe742bd0 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 291e17a2-364c-4981-ae31-bc59b7615093 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683e714ae76b4 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: 312dbe6c-5df1-46e7-b47b-08207c5b3a5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:10Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683e714af08da namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 4e6ae58b-b42b-41ba-9252-18df3533701c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 386bb619-5985-44dd-a988-1aefd39cdf65 kind: Event lastTimestamp: "2026-07-28T17:22:13Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:13Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6.18c683e7b0bda788 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 4f798a31-63b5-482d-bab5-1477b153804d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d0cd1-predictor-58d767b4ff namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 7653f4ad-98b3-45d5-8671-02c107b0f7d8 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: 'Created pod: success-200-isvc-d0cd1-predictor-58d767b4ff-s94d6' metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: success-200-isvc-d0cd1-predictor-58d767b4ff.18c683d12475f6e7 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 3ecf0ea0-ebd2-46ce-b615-4786af5bd043 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: d0b0d002-dad7-4ed5-94c6-f0c847015621 kind: Event lastTimestamp: "2026-07-28T17:20:36Z" message: Scaled up replica set success-200-isvc-d0cd1-predictor-58d767b4ff from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:20:36Z" name: success-200-isvc-d0cd1-predictor.18c683d122b4e834 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d078fd36-7f15-47cc-a35e-8177e3e01364 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: e6384fd6-cfd8-4fc8-88c2-e845ec3ea17f kind: Event lastTimestamp: "2026-07-28T17:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:21Z" name: success-200-isvc-d0cd1-predictor.18c683d4a10ebb59 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 5d83fd8e-946c-422f-a733-7b7b68ea762d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: e6384fd6-cfd8-4fc8-88c2-e845ec3ea17f kind: Event lastTimestamp: "2026-07-28T17:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:21Z" name: success-200-isvc-d0cd1-predictor.18c683d4a17471cb namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 99b50bd8-8182-4637-bbcb-f78a6f0f3361 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: e6384fd6-cfd8-4fc8-88c2-e845ec3ea17f kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:06Z" name: success-200-isvc-d0cd1-predictor.18c683d820ab0fea namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: de7ea086-8522-4682-bd89-7bf05ef7c473 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0cd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: e6384fd6-cfd8-4fc8-88c2-e845ec3ea17f kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:06Z" name: success-200-isvc-d0cd1-predictor.18c683d821c86f9f namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 8ff23131-8762-4416-b7e8-0c5e30fd4d02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0cd1 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: a8b33e0c-1e35-4e0c-9aaf-c138e91ba5fa kind: Event lastTimestamp: "2026-07-28T17:21:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:43Z" name: success-200-isvc-d0cd1.18c683d122bb7e68 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: ce4edf5a-7f80-4686-961b-5735b1ed201e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0cd1 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: a8b33e0c-1e35-4e0c-9aaf-c138e91ba5fa kind: Event lastTimestamp: "2026-07-28T17:21:43Z" message: InferenceService [success-200-isvc-d0cd1] is Ready metadata: creationTimestamp: "2026-07-28T17:21:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:43Z" name: success-200-isvc-d0cd1.18c683e0bd38f5c1 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 96358e2e-17cb-40db-ab2f-3371e2cdc91a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:00.884942Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e4209-predictor-5d46777c7f-24bwf to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a2327aecd namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: b0f4924f-f1bb-4823-adc9-ad8f3d18dbe1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a3d7f542e namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: 0ec706d6-c74f-48c2-addc-afcef2846b62 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a3ef02e40 namespace: kserve-ci-e2e-test resourceVersion: "26521" uid: 92bc8ff2-4de5-411a-a2bb-efe1d9898c7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a48369b5f namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 3c8dd384-99c4-4066-8047-251ae2b388e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a48bdda31 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 71d945f5-a467-430a-8564-03442fc62874 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a48c9d289 namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 7fc0c987-d87a-4ffb-9179-730dcca6c4e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a4c0f1fa0 namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: c6e07e51-05c5-4340-a846-af518814af2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:01Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a4c8437df namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: a43d5304-1cc9-4dce-855f-40f952923856 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:43:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T17:43:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:48Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c6850a971e3bf6 namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: 764b36f8-8b76-4e27-850a-8def6a79cbae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T18:00:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:00:30Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c685fe82bbeed2 namespace: kserve-ci-e2e-test resourceVersion: "34194" uid: b0c3b41b-db44-4b30-94c9-23ec57426cd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: b6906f40-d78c-436c-bd07-8d8b0ec81237 kind: Event lastTimestamp: "2026-07-28T18:00:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:00:30Z" name: success-200-isvc-e4209-predictor-5d46777c7f-24bwf.18c685fe82bc56c1 namespace: kserve-ci-e2e-test resourceVersion: "34198" uid: 89e1e11a-c913-4b2a-b7d3-ff0ad01e5456 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e4209-predictor-5d46777c7f namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 83e5713e-99cf-4059-86e2-8553540b931e kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: 'Created pod: success-200-isvc-e4209-predictor-5d46777c7f-24bwf' metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-e4209-predictor-5d46777c7f.18c6850a22972dbc namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: 416c1d95-b7a9-4fa2-ba7a-2f575c825ac4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 7ff60076-1c9c-42f9-adb5-be6012d83cdd kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: Scaled up replica set success-200-isvc-e4209-predictor-5d46777c7f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-e4209-predictor.18c6850a20b9e55b namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: d5780e2a-a4da-46a7-8937-3b066d0646a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: ef2ceffd-49dc-4c11-ac04-9924cff57626 kind: Event lastTimestamp: "2026-07-28T17:43:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:45Z" name: success-200-isvc-e4209-predictor.18c6850d9f35d6e8 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: 9f46caf6-d966-426f-b98b-a93b426ecc77 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: ef2ceffd-49dc-4c11-ac04-9924cff57626 kind: Event lastTimestamp: "2026-07-28T17:43:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:45Z" name: success-200-isvc-e4209-predictor.18c6850d9fac8722 namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: e6aae1c0-d865-46f1-a358-5f0a9a967af5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:44:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: ef2ceffd-49dc-4c11-ac04-9924cff57626 kind: Event lastTimestamp: "2026-07-28T17:44:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:30Z" name: success-200-isvc-e4209-predictor.18c685181d1cf7c4 namespace: kserve-ci-e2e-test resourceVersion: "27514" uid: 3740280d-7851-48ee-8b88-3c049c6eb616 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:44:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4209-predictor namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: ef2ceffd-49dc-4c11-ac04-9924cff57626 kind: Event lastTimestamp: "2026-07-28T17:44:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:30Z" name: success-200-isvc-e4209-predictor.18c685181d6c557a namespace: kserve-ci-e2e-test resourceVersion: "27515" uid: 44e011f1-0ab7-43ef-85b0-c4e6d81a00a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 45c6ef87-fffc-42bc-9380-1a4c74c2c75f kind: Event lastTimestamp: "2026-07-28T17:43:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:59Z" name: success-200-isvc-e4209.18c6850a20ac23de namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: 4356ffb4-8361-4001-9bc2-2eaebf940f78 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 45c6ef87-fffc-42bc-9380-1a4c74c2c75f kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-e4209": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e4209": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-e4209.18c6850a2159ff9c namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: 222b5080-8f98-4b4e-81ea-3d92e841bf4e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 45c6ef87-fffc-42bc-9380-1a4c74c2c75f kind: Event lastTimestamp: "2026-07-28T17:43:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e4209": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:00Z" name: success-200-isvc-e4209.18c6850a215a46b5 namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: fd586f83-5e1f-446b-a243-1ceb94a05e2e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4209 namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: 45c6ef87-fffc-42bc-9380-1a4c74c2c75f kind: Event lastTimestamp: "2026-07-28T17:43:59Z" message: InferenceService [success-200-isvc-e4209] is Ready metadata: creationTimestamp: "2026-07-28T17:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:59Z" name: success-200-isvc-e4209.18c68517ac707e9f namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: 7126eb62-c391-40e2-9742-b4a06b65fabc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:36.128971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fca3b-predictor-666b4997d9-54hbv to ip-10-0-136-80.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:36Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478accc0c70 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: bcae8129-65ea-430b-9c90-13f2b8767aca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fca3b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:36Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478b6aeeab9 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: 0de7e73f-25f5-456c-a7cf-8641cda875bd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478ecdac1e5 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: 773a1700-56af-41ab-9ab4-0316788a5efb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478ee3fb574 namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 2f0958d6-47b1-4b95-bb00-02089f469d08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478f26a61ec namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: be7c55f5-1b24-43fb-8698-b4b932d5bf8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478f2fa63d4 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 3e3c15ac-83d9-443f-90df-763a92aa8b9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478f302f780 namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: d304c60c-29cd-4054-aba4-6f9fa8ded3e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478f609ce52 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 381045b3-dab7-40e8-9ac8-f9f759270ba9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:32:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:37Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c68478f6834487 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 0f2e7036-002c-44f5-b148-485a662b9f68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:32:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:41:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:54Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c684792d05fc67 namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: cf6ec633-1cf9-4cfe-9e81-5f4977f41a9f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c684f9e042d92e namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: ebd8b57b-05a6-4e96-80bd-2bcff5783a4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:41:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:51Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c684f9e04382ce namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: 40d2d6ea-edde-420a-9721-9f397acea671 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: be9ccb9c-ebd3-44b2-b685-d5e260c772a9 kind: Event lastTimestamp: "2026-07-28T17:41:54Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:41:54Z" name: success-200-isvc-fca3b-predictor-666b4997d9-54hbv.18c684faa18d9a9f namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: 9d4cd1c9-d0f5-47d6-b6e2-e970f62c2b20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-80.ec2.internal source: component: kubelet host: ip-10-0-136-80.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fca3b-predictor-666b4997d9 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 5ccb6e1c-d8bd-4504-bd57-1b3ddf9d20d8 kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: 'Created pod: success-200-isvc-fca3b-predictor-666b4997d9-54hbv' metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:36Z" name: success-200-isvc-fca3b-predictor-666b4997d9.18c68478ac66dbe9 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 621dbdba-4e83-4f2e-b24f-c6e8f23a3f2d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: b7176331-6bc3-4670-880c-22f95b0a9b5e kind: Event lastTimestamp: "2026-07-28T17:32:36Z" message: Scaled up replica set success-200-isvc-fca3b-predictor-666b4997d9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:36Z" name: success-200-isvc-fca3b-predictor.18c68478aa8cee69 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 6faeb78d-6de4-4c89-bb0a-4399a2dc5ba3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 24ccfe50-68d2-4dfa-a4b4-30096610e2a4 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: success-200-isvc-fca3b-predictor.18c6847c292df15e namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 84885170-ec7d-41ae-a7bc-635505fc9674 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 24ccfe50-68d2-4dfa-a4b4-30096610e2a4 kind: Event lastTimestamp: "2026-07-28T17:33:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:36Z" name: success-200-isvc-fca3b-predictor.18c6847c29988f98 namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 46f4ac22-78bf-4d04-9a04-f9e613db1181 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 24ccfe50-68d2-4dfa-a4b4-30096610e2a4 kind: Event lastTimestamp: "2026-07-28T17:33:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:06Z" name: success-200-isvc-fca3b-predictor.18c6847fa92c6241 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 8d222ef4-9ad9-4b62-b538-2d0250e109e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fca3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 24ccfe50-68d2-4dfa-a4b4-30096610e2a4 kind: Event lastTimestamp: "2026-07-28T17:33:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:06Z" name: success-200-isvc-fca3b-predictor.18c6847fa9e56b79 namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: bbf5418b-d1d5-4df4-8cbf-9ffb352a3cdd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fca3b namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 0a37bc52-d8d7-41f0-a7d1-222952be1baf kind: Event lastTimestamp: "2026-07-28T17:33:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:24Z" name: success-200-isvc-fca3b.18c68478aaa04728 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 49abea8f-1dd0-4672-9ffa-3bd8655ae543 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fca3b namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 0a37bc52-d8d7-41f0-a7d1-222952be1baf kind: Event lastTimestamp: "2026-07-28T17:33:24Z" message: InferenceService [success-200-isvc-fca3b] is Ready metadata: creationTimestamp: "2026-07-28T17:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:24Z" name: success-200-isvc-fca3b.18c68483e8b35069 namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: 415bf4ef-c049-4329-bbbd-dc18d2ebf678 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:23:10.904325Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1fbfc-6cb7c7477-hpmmg to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:23:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:23:10Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c683f512c8802c namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 2af65d40-b1d7-42a3-8dea-16ea89a81906 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: "2026-07-28T17:23:11Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:23:11Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c683f52c50004a namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: d89dccfa-3a64-484a-94f5-1c9f0098c828 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1fbfc} kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: "2026-07-28T17:23:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:11Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c683f52d9d5ffd namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: bdec8dd3-5b25-4e6e-b63b-99eda5b5d2c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1fbfc} kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: "2026-07-28T17:23:11Z" message: 'Created container: switch-graph-1fbfc' metadata: creationTimestamp: "2026-07-28T17:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:11Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c683f5308c490b namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 4a72b046-e588-4eca-ac72-5b2575ff7ab5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1fbfc} kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: "2026-07-28T17:23:11Z" message: Started container switch-graph-1fbfc metadata: creationTimestamp: "2026-07-28T17:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:11Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c683f5311251e2 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: bc266b4d-d530-4b90-8fbd-ce99eb2ceb4c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:31:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1fbfc} kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: "2026-07-28T17:31:25Z" message: Stopping container switch-graph-1fbfc metadata: creationTimestamp: "2026-07-28T17:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:25Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c6846842b8141f namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 80ca41cc-c7f1-4db0-ae38-4a792d8909e4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1fbfc} kind: Pod name: switch-graph-1fbfc-6cb7c7477-hpmmg namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 86aed8ef-acff-44e0-b41b-2d88a4663691 kind: Event lastTimestamp: "2026-07-28T17:31:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:31:53Z" name: switch-graph-1fbfc-6cb7c7477-hpmmg.18c68468d4e9b80d namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: a70d6775-f9a7-4027-bfde-80ad89be3ad8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1fbfc-6cb7c7477 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: a7f95d78-ea45-4bbe-9bf5-4c2a9313cd4b kind: Event lastTimestamp: "2026-07-28T17:23:10Z" message: 'Created pod: switch-graph-1fbfc-6cb7c7477-hpmmg' metadata: creationTimestamp: "2026-07-28T17:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:10Z" name: switch-graph-1fbfc-6cb7c7477.18c683f512550d4f namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 51d7c5fc-a28c-400a-b585-b0f0f336cb92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 505c7bf0-81c8-484f-8d75-07f48154823a kind: Event lastTimestamp: "2026-07-28T17:23:10Z" message: Scaled up replica set switch-graph-1fbfc-6cb7c7477 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:10Z" name: switch-graph-1fbfc.18c683f511c31bea namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: d412cf0e-02c2-415c-9de7-26ce17464fe4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 609cdb09-78f9-478e-b86a-61a648c08d38 kind: Event lastTimestamp: "2026-07-28T17:23:10Z" message: InferenceGraph [switch-graph-1fbfc] is Ready metadata: creationTimestamp: "2026-07-28T17:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:10Z" name: switch-graph-1fbfc.18c683f517036ac0 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 080e0837-f572-4252-a34f-782e64258de2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:23:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0c4793af-fd82-45fb-90be-a7eee0642b51 kind: Event lastTimestamp: "2026-07-28T17:23:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:55Z" name: switch-graph-1fbfc.18c683f890224809 namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: fcddcfdd-291e-43fc-b68c-1b9b9bee5250 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:23:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1fbfc namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0c4793af-fd82-45fb-90be-a7eee0642b51 kind: Event lastTimestamp: "2026-07-28T17:23:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:55Z" name: switch-graph-1fbfc.18c683f890877d6a namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: e42ddd63-e748-4c1a-a1fd-c703cc9ce2f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:21:56.395150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d0cd1-54f4d5769d-ww4ps to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:21:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:21:56Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e3b9b0aed2 namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 542ca27a-49ce-4fd9-bb2b-b1574b8ce9c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:56Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:21:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d0cd1-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:56Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e3c18494c6 namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: cb4f484a-662e-4980-bcb0-0edfacd6791b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:57Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:21:57Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:21:57Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e3f6d6c4b3 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 799d9b0c-898a-4800-97a9-892de0456801 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0cd1} kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:21:57Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" metadata: creationTimestamp: "2026-07-28T17:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:57Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e3f82bbfaf namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: e6506289-ea53-4ca6-ab29-4adf2a1de549 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0cd1} kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:22:00Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" in 3.251s (3.251s including waiting). Image size: 218226779 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:00Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e4b9f3f6cc namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 3ad92996-d474-42e6-9729-286d5bbc4bc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0cd1} kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:22:00Z" message: 'Created container: switch-graph-d0cd1' metadata: creationTimestamp: "2026-07-28T17:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:00Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e4bd4b6f54 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: cb7a5670-5a28-4a02-a1f5-6b8bcbf56ce7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0cd1} kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:22:00Z" message: Started container switch-graph-d0cd1 metadata: creationTimestamp: "2026-07-28T17:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:00Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e4bdbbb6eb namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 4b141dba-fc1d-4c5a-aebf-0ef048b2e8c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0cd1} kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:22:10Z" message: Stopping container switch-graph-d0cd1 metadata: creationTimestamp: "2026-07-28T17:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:10Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e70b446a77 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 61acfd4f-71c4-4176-a8c4-db94590d0ef5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0cd1} kind: Pod name: switch-graph-d0cd1-54f4d5769d-ww4ps namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 5964eb4b-a46e-4073-9637-615c451cc548 kind: Event lastTimestamp: "2026-07-28T17:22:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:36Z" name: switch-graph-d0cd1-54f4d5769d-ww4ps.18c683e74e6594ba namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 82db3a5f-b42c-4691-859a-1bf04dedbb8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d0cd1-54f4d5769d namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: af7a767c-76fc-4557-9e6a-c4f738ad6712 kind: Event lastTimestamp: "2026-07-28T17:21:56Z" message: 'Created pod: switch-graph-d0cd1-54f4d5769d-ww4ps' metadata: creationTimestamp: "2026-07-28T17:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:56Z" name: switch-graph-d0cd1-54f4d5769d.18c683e3b94f3265 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 57a6d65a-5f75-4ba3-a39a-c41f6e4ce940 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d0cd1 namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 04ba9312-4222-4e30-be5f-45e20a0e6623 kind: Event lastTimestamp: "2026-07-28T17:21:56Z" message: Scaled up replica set switch-graph-d0cd1-54f4d5769d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:56Z" name: switch-graph-d0cd1.18c683e3b8544ba2 namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: b9501107-dd19-4dbe-9805-2bfe03fba7fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d0cd1 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: d6825a3c-5171-4aac-93a7-4172d0837cb2 kind: Event lastTimestamp: "2026-07-28T17:21:56Z" message: InferenceGraph [switch-graph-d0cd1] is Ready metadata: creationTimestamp: "2026-07-28T17:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:56Z" name: switch-graph-d0cd1.18c683e3bc0e6904 namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: ae6a6de9-977f-4eee-a839-bc3348afd882 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:10.937483Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27330" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e4209-8484bbd765-5lcd6 to ip-10-0-134-14.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:10Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c6851a729ca14f namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: dfa39e1a-9887-4afa-b838-1584e3b48fff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5b5cdff8-g626j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T17:44:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e4209-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:11Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c6851a7df11c43 namespace: kserve-ci-e2e-test resourceVersion: "27345" uid: 04f282bc-4a16-4744-8679-2c40c563da30 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T17:44:11Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:11Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c6851aafeb2f72 namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: 94abfc3d-896c-48c2-b85b-c3abfcd4d7a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e4209} kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T17:44:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:11Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c6851ab1465b57 namespace: kserve-ci-e2e-test resourceVersion: "27358" uid: 7ec5ac70-9e9a-4849-ace7-25aac28ebb96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e4209} kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T17:44:12Z" message: 'Created container: switch-graph-e4209' metadata: creationTimestamp: "2026-07-28T17:44:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:12Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c6851ab443844e namespace: kserve-ci-e2e-test resourceVersion: "27359" uid: a4673950-0d2d-435b-880c-3a63130f6bb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e4209} kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T17:44:12Z" message: Started container switch-graph-e4209 metadata: creationTimestamp: "2026-07-28T17:44:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:12Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c6851ab4c69dd1 namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: 584db300-909c-4f09-bd4c-c78917ff3ce2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e4209} kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T18:00:30Z" message: Stopping container switch-graph-e4209 metadata: creationTimestamp: "2026-07-28T18:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:00:30Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c685fe75d96eab namespace: kserve-ci-e2e-test resourceVersion: "34178" uid: da4a721b-0489-4a54-9992-d68722b55fc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T18:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e4209} kind: Pod name: switch-graph-e4209-8484bbd765-5lcd6 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: a57c3967-9b51-4cfa-b639-1c54ceda95fa kind: Event lastTimestamp: "2026-07-28T18:00:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T18:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:00:53Z" name: switch-graph-e4209-8484bbd765-5lcd6.18c685ff37305924 namespace: kserve-ci-e2e-test resourceVersion: "34420" uid: 13c5d177-c053-4c60-b8fc-875e515cf19b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-14.ec2.internal source: component: kubelet host: ip-10-0-134-14.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e4209-8484bbd765 namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: de247744-5b23-47a5-bdfa-e9bb72e21f5b kind: Event lastTimestamp: "2026-07-28T17:44:10Z" message: 'Created pod: switch-graph-e4209-8484bbd765-5lcd6' metadata: creationTimestamp: "2026-07-28T17:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:10Z" name: switch-graph-e4209-8484bbd765.18c6851a723237a1 namespace: kserve-ci-e2e-test resourceVersion: "27331" uid: cf2c710d-c9f4-4a57-b62c-8a24d265f6dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e4209 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: c5572741-8fd5-4ae8-8112-c33bc5f68b2b kind: Event lastTimestamp: "2026-07-28T17:44:10Z" message: Scaled up replica set switch-graph-e4209-8484bbd765 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:10Z" name: switch-graph-e4209.18c6851a7189b11a namespace: kserve-ci-e2e-test resourceVersion: "27328" uid: 50630d33-18aa-4fd8-9441-a2ba876479e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e4209 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: 7dc4f63a-aaa1-47a5-8cef-85b627e3989b kind: Event lastTimestamp: "2026-07-28T17:44:11Z" message: InferenceGraph [switch-graph-e4209] is Ready metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:11Z" name: switch-graph-e4209.18c6851a77451ef1 namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: 7faa7a82-143d-4ad0-974e-44e8ea60b16d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e4209 namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: a4830b32-e2e8-414e-bc9c-435999ff874e kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:40Z" name: switch-graph-e4209.18c6851df099a826 namespace: kserve-ci-e2e-test resourceVersion: "27591" uid: ad1be14f-6e76-45be-8465-fe5ae499880b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e4209 namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: a4830b32-e2e8-414e-bc9c-435999ff874e kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:40Z" name: switch-graph-e4209.18c6851df0ea32ee namespace: kserve-ci-e2e-test resourceVersion: "27592" uid: 97a7ebf2-090b-4e69-b4f8-67398b8a0471 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34450"