--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:58:03.189323Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-109dc-6bb57b867f-prbqj to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:58:03Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efaadcb3557 namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 1fb82c2d-fc64-41cf-bc77-042be0088300 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-109dc-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efabaf94117 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 04200e4d-4f92-421f-a2c5-30e599a35caa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:05Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:05Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:58:05Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efb214fff06 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 163acaa4-a627-4f6c-9ad5-e152ab7f9da0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-109dc} kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:05Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efb22c9c3b8 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 4a2f5ffd-bf2d-46f2-9b8e-68373b342e71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-109dc} kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:05Z" message: 'Created container: ensemble-graph-109dc' metadata: creationTimestamp: "2026-07-08T16:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:05Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efb262f3e34 namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 93f8d5c1-04df-4cef-ad87-18a652e63aa0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-109dc} kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:05Z" message: Started container ensemble-graph-109dc metadata: creationTimestamp: "2026-07-08T16:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:05Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efb26a91582 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 803ac78d-1e14-4375-9bb3-5a402b3524ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-109dc} kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Stopping container ensemble-graph-109dc metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efd05ef16fa namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: e67d091f-c459-411a-823f-5005a2566339 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-109dc} kind: Pod name: ensemble-graph-109dc-6bb57b867f-prbqj namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d2f9b0eb-2a9a-4fd7-8eb9-960e3519caa1 kind: Event lastTimestamp: "2026-07-08T16:58:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:58:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:41Z" name: ensemble-graph-109dc-6bb57b867f-prbqj.18c05efdc3c6a7fd namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 2dd10cf1-c2e8-45da-9ed7-84d0b71e7c74 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-109dc-6bb57b867f namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 4ba1fe55-0cc2-4590-86f0-e1097e0e820b kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'Created pod: ensemble-graph-109dc-6bb57b867f-prbqj' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: ensemble-graph-109dc-6bb57b867f.18c05efaad4ecd4f namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 88d60194-f1b3-436d-93b0-ea098a75144a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-109dc namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 39568277-64a7-47cb-9ea1-e736e0521f88 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Scaled up replica set ensemble-graph-109dc-6bb57b867f from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: ensemble-graph-109dc.18c05efaac8af27c namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: d610ab4c-6592-40e6-b4a2-50f5f2608c87 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-109dc namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 07f7204f-4298-4c74-ab7b-9fd66c0c722d kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: InferenceGraph [ensemble-graph-109dc] is Ready metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:03Z" name: ensemble-graph-109dc.18c05efab2f60f82 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: a995a33a-81f1-460a-8e6e-2ee39ad4f5b2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:59:13.508847Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-806a8-d67b8df85-lk848 to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f0b0d2a4b0a namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: afb465af-598e-456d-8398-3240939026d5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f0b26b2b08b namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: cd4979a2-617c-4c66-85dc-5613af74f1c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-806a8} kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f0b27d90f2d namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: 3508496b-36b6-4501-ae9d-a9d4b2767d68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-806a8} kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: 'Created container: ensemble-graph-806a8' metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f0b2a8f2b04 namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 17a14f2a-3ea6-43ab-820c-9d462134b85b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-806a8} kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: Started container ensemble-graph-806a8 metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f0b2b05c07f namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: c045e693-e796-42a9-bb13-aa6c0ee75db8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-806a8} kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Stopping container ensemble-graph-806a8 metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f7e3472ef24 namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: ebc4ad1b-e087-40e9-8e29-4ce2bb0d6868 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-806a8} kind: Pod name: ensemble-graph-806a8-d67b8df85-lk848 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: dde80ee5-3058-43da-b7f4-0ff1557cb67d kind: Event lastTimestamp: "2026-07-08T17:07:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:55Z" name: ensemble-graph-806a8-d67b8df85-lk848.18c05f7ed180cbe3 namespace: kserve-ci-e2e-test resourceVersion: "26225" uid: 960c5e28-260b-4828-80a2-ba2f5bb4e575 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-806a8-d67b8df85 namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 215d107b-0cba-4216-9cef-d785e05da97d kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Created pod: ensemble-graph-806a8-d67b8df85-lk848' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8-d67b8df85.18c05f0b0c966157 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: a729a83c-1fc8-4ea0-873e-c7e200d6d313 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-806a8 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: d8084733-4f09-46cd-91a4-beee3ee9827b kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Scaled up replica set ensemble-graph-806a8-d67b8df85 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8.18c05f0b0bbe8497 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: e8429384-ae34-426a-a03c-f9942a455438 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-806a8 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: 5d5ece93-5ed5-4902-86bc-2ca1b5d306a4 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: InferenceGraph [ensemble-graph-806a8] is Ready metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8.18c05f0b127f2c43 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: a2635e70-c3cf-43ce-acb4-e98f6def5f2e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-806a8 namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: 5d5ece93-5ed5-4902-86bc-2ca1b5d306a4 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-806a8": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-806a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8.18c05f0b153d71c3 namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 2097f686-3dec-43c3-9ff5-775f7692f815 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-806a8 namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: 5d5ece93-5ed5-4902-86bc-2ca1b5d306a4 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-806a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: ensemble-graph-806a8.18c05f0b153dc254 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: a17803b0-1387-406f-80ae-7dce338401e8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-806a8 namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 577cf373-2001-48c8-b7f6-030c52b58d24 kind: Event lastTimestamp: "2026-07-08T16:59:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:28Z" name: ensemble-graph-806a8.18c05f0e8a1ef597 namespace: kserve-ci-e2e-test resourceVersion: "22203" uid: 5a9c2485-40da-4bf5-9e06-64b7fc3b50fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-806a8 namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 577cf373-2001-48c8-b7f6-030c52b58d24 kind: Event lastTimestamp: "2026-07-08T16:59:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:28Z" name: ensemble-graph-806a8.18c05f0e8a99b6da namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: b2948141-717c-4e7f-b65b-6103de4edcd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:57:03.187191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-109dc-predictor-5dbbcdb786-92l9g to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecb5636821 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 4a0d1c74-1e38-4316-aef5-783acf7ef510 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eeccf5a1ad1 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: 93f5c9b2-4b73-4447-a7b6-ea7628bdb145 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecd129df50 namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: 6b382c88-b234-49ee-925c-0a5d7ff12718 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecd35b6413 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: bc12c78a-2d25-476d-95c7-1d5d113b1a1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecd3c7e2d7 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 23f25139-ab77-46d0-b5ee-d651e34aa389 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecd3d01b61 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 3e4246ba-7970-4eea-b1bb-eff597a48c8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecd613994c namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: d454762a-e8aa-470f-9fa8-b64282230bc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecd67f9b84 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: c32d65cf-7872-47ff-8c07-8f80dd375d57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:57:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:40Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05eecf41ee762 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: d1028b09-cce8-4543-b222-54dd6f61ad08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05efd0b9b6f97 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 882861cd-b4fb-461d-ab37-e369abbe2a98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05efd0b9bebef namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: b1b1c8fc-d443-4999-80d9-ea38344d1cd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 415c8967-85e5-46d7-a26a-c92300dbef49 kind: Event lastTimestamp: "2026-07-08T16:58:15Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:15Z" name: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g.18c05efd7ca0d321 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 5258393d-57fe-475a-ab93-5e478f2e47c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-109dc-predictor-5dbbcdb786 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 1c58292d-76d1-4edc-a099-797d50d36ecc kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Created pod: error-404-isvc-109dc-predictor-5dbbcdb786-92l9g' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor-5dbbcdb786.18c05eecb4ca1cf6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: ea615f7f-4b0f-4be2-8104-cd70c87c8697 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 5c70c13b-6468-4aec-b0c9-998476225303 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Scaled up replica set error-404-isvc-109dc-predictor-5dbbcdb786 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc-predictor.18c05eecb3305cff namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: f1040dc0-f4d1-46b0-b6d8-a83dec335cf6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 0f13dd25-69ad-4893-8931-cb4917ac0c42 kind: Event lastTimestamp: "2026-07-08T16:57:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:33Z" name: error-404-isvc-109dc-predictor.18c05ef031a98bb2 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 1412a079-6e37-4ee7-bec5-7428c6aaecd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 0f13dd25-69ad-4893-8931-cb4917ac0c42 kind: Event lastTimestamp: "2026-07-08T16:57:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:33Z" name: error-404-isvc-109dc-predictor.18c05ef032186d90 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: b8776ee2-8cf8-49df-91fa-202b45618d6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 0f13dd25-69ad-4893-8931-cb4917ac0c42 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-109dc-predictor.18c05ef72fc6aaa0 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 8a7c29b3-9801-4d17-a6ab-cab70e8588d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 0f13dd25-69ad-4893-8931-cb4917ac0c42 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-109dc-predictor.18c05ef73031df35 namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: cf079a04-940b-4c14-9bf4-80cee448f373 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: de634b09-21d9-4301-bdc7-0a1aa4fe7c9b kind: Event lastTimestamp: "2026-07-08T16:57:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:50Z" name: error-404-isvc-109dc.18c05eecb3852d7b namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 04f4f051-f2dc-4ed4-a038-871f0f8b2727 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: de634b09-21d9-4301-bdc7-0a1aa4fe7c9b kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-109dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-109dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc.18c05eecb3d7254e namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: cfeec7e9-5ec9-4b6e-b60e-3a288802c0cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: de634b09-21d9-4301-bdc7-0a1aa4fe7c9b kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-109dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-109dc.18c05eecb3d76d5b namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 45e634d1-a9e3-411d-9d7c-abba1211248e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: de634b09-21d9-4301-bdc7-0a1aa4fe7c9b kind: Event lastTimestamp: "2026-07-08T16:57:50Z" message: InferenceService [error-404-isvc-109dc] is Ready metadata: creationTimestamp: "2026-07-08T16:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:50Z" name: error-404-isvc-109dc.18c05ef7b207cf50 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 4ba59029-c4a2-4cda-a8b3-5f5c181f58c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:58:03.079308Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-178bc-predictor-79b66c889d-d8dvk to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efaa73c96df namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 2aee0214-1085-4a01-8170-4dba5b773737 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac183ad46 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: ddcddd0e-5d9c-4389-b1c9-eabd4502da2e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac2c2e4d2 namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: 150d24b6-c90e-43ef-94f5-a751863a21f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac57c2e0c namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: 6b4407e1-b3bf-4b14-8b56-e85b29d4af9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac5efd0cf namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: be4d8f01-12ab-4a3c-9d13-3e2984200f3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac5f90586 namespace: kserve-ci-e2e-test resourceVersion: "21070" uid: 07c9c08f-628a-4282-b781-587743623f55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac8b8a6eb namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: 5443644b-c870-4a64-99b7-a5961c9a6b5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efac923fb5e namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: 14776a36-065c-40ed-b6a4-cafa41d65ad7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:58:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:40Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05efafe4d2035 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: 7a0be918-b53a-4760-b481-d85f2b0c8686 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05f0af86e8852 namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: af7f56c1-9b9c-4c75-bbb0-1fc4f751a435 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05f0af86f0b22 namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: ad520458-066a-4c62-a938-9ab7b8d3410f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: dcd51f3a-9e78-4902-887f-fb6bcda9b5a6 kind: Event lastTimestamp: "2026-07-08T16:59:15Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:15Z" name: error-404-isvc-178bc-predictor-79b66c889d-d8dvk.18c05f0b86822694 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 976e428a-568f-4de1-9968-c03c5dcd600f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-178bc-predictor-79b66c889d namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 61e49ee3-b04c-40d9-9f45-0c48e2a6db23 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'Created pod: error-404-isvc-178bc-predictor-79b66c889d-d8dvk' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor-79b66c889d.18c05efaa67fcc71 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: dd76b435-051d-4da1-a9e9-ec28e0a7977e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: c3bed6a0-228c-49f6-90f6-78cb1b0b1e6c kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Scaled up replica set error-404-isvc-178bc-predictor-79b66c889d from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc-predictor.18c05efaa4578cb0 namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: b1701de9-31a0-4cf9-808c-0d601063720f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 2b1509c7-55ee-4fe2-8256-77342f805739 kind: Event lastTimestamp: "2026-07-08T16:58:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:33Z" name: error-404-isvc-178bc-predictor.18c05efe22dd416c namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 524eb436-3f95-4709-8e09-eac0ef09a2cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 2b1509c7-55ee-4fe2-8256-77342f805739 kind: Event lastTimestamp: "2026-07-08T16:58:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:33Z" name: error-404-isvc-178bc-predictor.18c05efe235dd618 namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: fe224c10-5b3d-476f-a79a-6bae500c0b0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 2b1509c7-55ee-4fe2-8256-77342f805739 kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:58:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:03Z" name: error-404-isvc-178bc-predictor.18c05f0521259e76 namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: 8178d8c3-eb17-4a35-9ee2-65365f642947 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 2b1509c7-55ee-4fe2-8256-77342f805739 kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:58:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:03Z" name: error-404-isvc-178bc-predictor.18c05f05219f6c01 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 07bec6f6-6c7c-439f-abd6-8ec914a08685 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-178bc namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 497e5652-a9b3-4bbb-82cc-a76d481829e3 kind: Event lastTimestamp: "2026-07-08T16:58:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:50Z" name: error-404-isvc-178bc.18c05efaa45a0d71 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: 68960d32-f70a-4cf6-b4dd-4aebcb8fa829 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-178bc namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 497e5652-a9b3-4bbb-82cc-a76d481829e3 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-178bc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-178bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc.18c05efaa50add87 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: b4dbfbfb-8f20-4ae3-9333-242a394c7f9c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-178bc namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 497e5652-a9b3-4bbb-82cc-a76d481829e3 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-178bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:03Z" name: error-404-isvc-178bc.18c05efaa50b2de5 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: a2565d52-716b-4306-b54c-724f2aecfa9a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-178bc namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 497e5652-a9b3-4bbb-82cc-a76d481829e3 kind: Event lastTimestamp: "2026-07-08T16:58:50Z" message: InferenceService [error-404-isvc-178bc] is Ready metadata: creationTimestamp: "2026-07-08T16:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:50Z" name: error-404-isvc-178bc.18c05f05bc2fcb5c namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: 603fdb2a-cbc1-49dc-b301-ae6c5493a702 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:07:28.335781Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2c597-predictor-6bdcb48c76-7578d to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e4325d724 namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: ae61332d-b894-4600-bb60-116e7d4b40a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e5de7461e namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: d9d572b8-af9f-40cb-adf8-ea56a58d3228 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e5f43e2f4 namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 555bf92b-d848-4332-afdc-e106aa0cffcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e61ede905 namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 7bdf5f72-ccb7-4a67-b68a-146b99965a34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e625a4d13 namespace: kserve-ci-e2e-test resourceVersion: "25999" uid: 614cb180-cb6a-4e91-b41c-b23d8cb830d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e62642d41 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 7bf56a29-105a-42bb-be13-7c6b291164c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e65b3e96f namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 16f2596e-95f2-44e8-8701-6e42c3c345da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e662131a2 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 6131515c-ceb5-4863-8652-1d74f3d850bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:08:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:05Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f7e6c0d0846 namespace: kserve-ci-e2e-test resourceVersion: "26296" uid: 65288c3f-ec35-4bdc-9fa3-3cfe992a31bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f8e9af1853f namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: db1948b3-8b77-4846-a575-5402eb97574b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f8e9af21339 namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: 2d8d30cc-ba0b-44c0-95bc-4ba7dc7ba2fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 4e911cc7-1bab-45a3-95a8-d6b37031c1c7 kind: Event lastTimestamp: "2026-07-08T17:08:40Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:40Z" name: error-404-isvc-2c597-predictor-6bdcb48c76-7578d.18c05f8ef4331afc namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: e8352b00-95cf-433a-b1d1-97d9dc4642fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2c597-predictor-6bdcb48c76 namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 9f58d6dc-8ca4-4bf0-bbd8-18a5e6888f1a kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Created pod: error-404-isvc-2c597-predictor-6bdcb48c76-7578d' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor-6bdcb48c76.18c05f7e42877ff5 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: fcd413d9-b110-494e-870e-f1fe629a3aef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: 14504f9c-7845-4112-b902-5c3c2906b3a2 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Scaled up replica set error-404-isvc-2c597-predictor-6bdcb48c76 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597-predictor.18c05f7e40fae33c namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: ba7ba8e8-8b95-418c-8c5a-b7f16e9cb86c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "25950" uid: 7734da59-655b-43fd-9d8f-64cac69bc729 kind: Event lastTimestamp: "2026-07-08T17:07:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:58Z" name: error-404-isvc-2c597-predictor.18c05f81bf73fdb5 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: cb7d75ac-0d4a-4ac1-b1aa-890173c5512a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "25950" uid: 7734da59-655b-43fd-9d8f-64cac69bc729 kind: Event lastTimestamp: "2026-07-08T17:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:58Z" name: error-404-isvc-2c597-predictor.18c05f81bfb484e6 namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 044a0ffd-d203-4efd-8a35-7a6efd99a3ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: 7734da59-655b-43fd-9d8f-64cac69bc729 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-2c597-predictor.18c05f88bd58e3d5 namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: b23b3296-c93b-458e-9c09-f1c27977ab28 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: 7734da59-655b-43fd-9d8f-64cac69bc729 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-2c597-predictor.18c05f88bdbedab0 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: dfd41cd4-8c32-4754-8e9e-48573236acfa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c597 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: c2707e83-9c58-4caa-8e4c-cc1b9af8e0dc kind: Event lastTimestamp: "2026-07-08T17:08:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:15Z" name: error-404-isvc-2c597.18c05f7e40fd4193 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: 3e126878-c2be-41c4-a307-a5f667517981 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c597 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: c2707e83-9c58-4caa-8e4c-cc1b9af8e0dc kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-2c597": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2c597": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597.18c05f7e419e7dfd namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: d9904b27-6d1f-4ed6-920b-c0d2f5722a5e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c597 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: c2707e83-9c58-4caa-8e4c-cc1b9af8e0dc kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2c597": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-2c597.18c05f7e419ed4c3 namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: aa6120ca-0d61-46c5-ae99-2067d4a4c154 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2c597 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: c2707e83-9c58-4caa-8e4c-cc1b9af8e0dc kind: Event lastTimestamp: "2026-07-08T17:08:15Z" message: InferenceService [error-404-isvc-2c597] is Ready metadata: creationTimestamp: "2026-07-08T17:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:15Z" name: error-404-isvc-2c597.18c05f8927913b5e namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 340890cb-dd01-4ee3-be13-4a97bd6579a4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:47:48.261691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-429af-predictor-5679c5df6-n85zs to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6b814042ee namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 7331f25d-4267-4236-a813-fb326f02fcbf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6b9ab96fc2 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: f5012517-aac1-4290-9b65-79efe54362ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6b9bec6a0b namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0c6b17f1-6c74-419d-bda6-050af65203b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6b9eadf0f5 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: a0d18483-2e5c-4206-83b0-dfb10376369f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6b9f58b715 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: f48404a2-8d8d-45e1-88e1-f1de602c08d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6b9f613fe2 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 920762b9-c842-4e14-8321-f1ab090d40dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6ba1ec96fc namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: ab842976-73b5-45e0-9699-70fdaa1b0ed0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6ba24b1e32 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 59a15630-3b7a-4705-8654-daea8c5c4ffa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:57:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:05Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05e6be10d939e namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: d2ae7244-bff7-42dc-8a50-065a05e76b71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05eecb4a234da namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: d683641b-7ba4-4fed-bb43-fdc75a2fafd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05eecb4a7ee5a namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: ae5ce2fc-4615-4b2d-a5a8-7fa2fcd5b755 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-429af-predictor-5679c5df6-n85zs namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 678b5684-6ab2-4465-9ac7-78f25661f96c kind: Event lastTimestamp: "2026-07-08T16:57:05Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:05Z" name: error-404-isvc-429af-predictor-5679c5df6-n85zs.18c05eed556d5567 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 3c5071ed-2173-42a2-b52e-4e384c004778 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-429af-predictor-5679c5df6 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 48ac655d-5560-47dd-8844-b930bc4832b5 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Created pod: error-404-isvc-429af-predictor-5679c5df6-n85zs' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor-5679c5df6.18c05e6b80dcdeed namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 36c57db7-8beb-45a6-8593-49919e215904 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: 340daa4b-8553-4a16-9d2a-dfe770f0d920 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Scaled up replica set error-404-isvc-429af-predictor-5679c5df6 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af-predictor.18c05e6b7f3c587f namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9ab1d1ba-538d-46e2-8044-8eecff330762 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:48:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: e961a72a-cfad-44aa-91b6-a3b5536191ff kind: Event lastTimestamp: "2026-07-08T16:48:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:18Z" name: error-404-isvc-429af-predictor.18c05e6efdbccb32 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: 8d85d5d2-a014-4eb1-8837-c5202a4fa9ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:48:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: e961a72a-cfad-44aa-91b6-a3b5536191ff kind: Event lastTimestamp: "2026-07-08T16:48:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:18Z" name: error-404-isvc-429af-predictor.18c05e6efe299b64 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 703621fc-ab66-4d72-8e2d-19c6ec5051f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:48:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: e961a72a-cfad-44aa-91b6-a3b5536191ff kind: Event lastTimestamp: "2026-07-08T16:49:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:03Z" name: error-404-isvc-429af-predictor.18c05e75fc089be1 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: ff0d6252-8fd3-4ed0-816f-41b00f0539e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:48:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: e961a72a-cfad-44aa-91b6-a3b5536191ff kind: Event lastTimestamp: "2026-07-08T16:49:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:03Z" name: error-404-isvc-429af-predictor.18c05e75fc846552 namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: 5ac8f468-bdeb-4bb8-bc25-50f9f8ad628a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: a8f0504a-23d3-443f-9277-7202656f81ae kind: Event lastTimestamp: "2026-07-08T16:48:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:36Z" name: error-404-isvc-429af.18c05e6b7f42877d namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: a6b9911f-97dd-4466-a1ae-e5d163b33736 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: a8f0504a-23d3-443f-9277-7202656f81ae kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-429af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-429af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af.18c05e6b7fb97a0e namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 7419dcdc-2990-4bad-b979-49125b8e7ee5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: a8f0504a-23d3-443f-9277-7202656f81ae kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-429af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-429af.18c05e6b7fb9d73b namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: a6d61cde-9a23-4603-9d68-024cfab71e8b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: a8f0504a-23d3-443f-9277-7202656f81ae kind: Event lastTimestamp: "2026-07-08T16:48:35Z" message: InferenceService [error-404-isvc-429af] is Ready metadata: creationTimestamp: "2026-07-08T16:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:35Z" name: error-404-isvc-429af.18c05e769cceb32b namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: cd15c58d-45f6-4469-93c6-91fd9ba59fff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:58:13.538597Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-806a8-predictor-5595d45646-t22t9 to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd16a8b1c2 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 672bcabb-7c85-46c4-8abe-bde2310ded9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-806a8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd2055a4b8 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: a9462285-fc36-448e-8697-74bcb88f543c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd540d58f9 namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: ed2020e2-25fb-4308-94ad-245a891382e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd556476d3 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: aa5d3b27-0b08-45f4-92c6-698489230dad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd58bcfa05 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 67ceda87-c92e-4b48-bc90-6d5429593817 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd5928a12d namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: f00522a9-7e42-46d5-bd17-ac07f1e16ec2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd5932cf7c namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: 0a0b6d47-fa8a-4764-946d-6e2cf7e2109d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd5bfbecd6 namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c9b11f73-79cc-40fe-b9a9-e89bd3d5e802 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efd5c66b91d namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: a5545779-6ca3-4620-af74-0437129c3ac1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T16:58:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:52Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05efdce1b788e namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: fb033981-4c79-4aa7-92ce-bfa86b37cd83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05f7e3e61f448 namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: 10236627-e714-4066-bea2-75387624c1fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-806a8-predictor-5595d45646-t22t9 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: d176335b-e0f2-4943-8d7d-4bd699d52418 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: error-404-isvc-806a8-predictor-5595d45646-t22t9.18c05f7e3e62628a namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: 4bd63672-76af-4856-87ec-332a3b82b65a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-806a8-predictor-5595d45646 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 0519358e-9a8d-49f9-ae65-d4c9656bdd6a kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'Created pod: error-404-isvc-806a8-predictor-5595d45646-t22t9' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-806a8-predictor-5595d45646.18c05efd16487aaf namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 33a1be90-a8f4-4b1e-933f-b4136d620b39 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 339a3e33-e72d-4da7-8c09-cda101ebbf8f kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Scaled up replica set error-404-isvc-806a8-predictor-5595d45646 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-806a8-predictor.18c05efd1472e3a5 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 6ad3c7fa-6e01-4bac-8f3a-15340859a4fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: 5c055a27-3180-4d6e-8bb0-cefb0e885767 kind: Event lastTimestamp: "2026-07-08T16:58:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:28Z" name: error-404-isvc-806a8-predictor.18c05f0093269807 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: adc5ebbd-d78a-4d77-b56f-4a63d06d586e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: 5c055a27-3180-4d6e-8bb0-cefb0e885767 kind: Event lastTimestamp: "2026-07-08T16:58:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:28Z" name: error-404-isvc-806a8-predictor.18c05f0093a52d65 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: a945c6f1-f813-43a1-ae5d-ab3bda8cac59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:58:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 5c055a27-3180-4d6e-8bb0-cefb0e885767 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-806a8-predictor.18c05f0412a83c19 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: 4b55fd8d-fad3-4628-9465-44bad3a89003 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:58:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 5c055a27-3180-4d6e-8bb0-cefb0e885767 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-806a8-predictor.18c05f04132776fc namespace: kserve-ci-e2e-test resourceVersion: "22031" uid: ee5cc32c-698e-4d44-8825-02e7bb323a35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: f5b87832-250e-4a4e-947e-519d2f146ffd kind: Event lastTimestamp: "2026-07-08T16:59:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:02Z" name: error-404-isvc-806a8.18c05efd146b4b2f namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 8a52f047-1cf5-4d93-8132-c46576692c1a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: f5b87832-250e-4a4e-947e-519d2f146ffd kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-806a8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-806a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-806a8.18c05efd14bb40a6 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: a737b68c-8363-412e-a79e-a30dff6a90d0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: f5b87832-250e-4a4e-947e-519d2f146ffd kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-806a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:13Z" name: error-404-isvc-806a8.18c05efd14bbe03f namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 51a7ebd8-a578-48a4-9525-bc4e0a08e7dc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: f5b87832-250e-4a4e-947e-519d2f146ffd kind: Event lastTimestamp: "2026-07-08T16:59:02Z" message: InferenceService [error-404-isvc-806a8] is Ready metadata: creationTimestamp: "2026-07-08T16:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:02Z" name: error-404-isvc-806a8.18c05f08893a0b69 namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: ad986288-07da-4328-a2b7-068936905c92 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:08:38.754777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ab489-predictor-76897c7688-74fs9 to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ea8728449 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 4cbdd378-a00f-452c-9891-f903d57adca6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26852" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ec2282f59 namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: 93c02731-cc3f-4686-92b8-b3eca94d43a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ec372495f namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: 49f14944-32e4-48ee-ad8c-5a206eeb8ff1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ec646df5f namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: bdbba57f-29e9-4327-b0f8-46bf193d1a9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ec6b309a0 namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 2df33bb2-7096-4a6a-8dc1-09cd008275a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ec6bd1ab0 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 0f86430c-7d6c-4661-86b7-c077112dd9fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ec9b42138 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: e91985d1-60c6-4ae2-bf1e-f248b2b7af19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8eca21c0b4 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 672bb2e0-be60-4a7b-b71c-bb672a353f98 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:18:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:05Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c05f8ecc218258 namespace: kserve-ci-e2e-test resourceVersion: "31072" uid: 9436c05c-9a46-4a7b-ab16-e8b4e369d10f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:18:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:03Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c06012227c9ffe namespace: kserve-ci-e2e-test resourceVersion: "31050" uid: 1c856e1f-74a3-4686-9840-fd719fb3673e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:18:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:03Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c06012227d18f9 namespace: kserve-ci-e2e-test resourceVersion: "31051" uid: fbc6641c-88e2-4952-a544-4d5c1d3e9635 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:18:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab489-predictor-76897c7688-74fs9 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 2c2ba3ac-69f2-47cb-937b-a54278f1ed13 kind: Event lastTimestamp: "2026-07-08T17:18:05Z" message: 'Readiness probe failed: Get "https://10.133.0.48:8643/healthz": dial tcp 10.133.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:05Z" name: error-404-isvc-ab489-predictor-76897c7688-74fs9.18c0601294c1ee32 namespace: kserve-ci-e2e-test resourceVersion: "31070" uid: deb38653-870d-45a8-9308-7b8d932fd25f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ab489-predictor-76897c7688 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 5a7e9f65-bf82-4ae7-8186-2b35fbb4184f kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'Created pod: error-404-isvc-ab489-predictor-76897c7688-74fs9' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-ab489-predictor-76897c7688.18c05f8ea763a610 namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 177e6d01-4c7f-42ab-8bd3-99f3490f6486 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: de19128c-6e77-4b12-9f25-ea8616c77ade kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Scaled up replica set error-404-isvc-ab489-predictor-76897c7688 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-ab489-predictor.18c05f8ea4272d70 namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: 74a6914e-9966-4823-8040-d8d3c0ebdbe3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:08:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: 45f00b94-4f6e-4c77-b0d1-7acbf718c59e kind: Event lastTimestamp: "2026-07-08T17:09:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:38Z" name: error-404-isvc-ab489-predictor.18c05f92229783e5 namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: 0ea973c2-5bbb-47b3-9cd2-f341c9b8b325 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:08:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: 45f00b94-4f6e-4c77-b0d1-7acbf718c59e kind: Event lastTimestamp: "2026-07-08T17:09:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:38Z" name: error-404-isvc-ab489-predictor.18c05f922307818f namespace: kserve-ci-e2e-test resourceVersion: "27456" uid: 5f398763-4f86-464e-b4c7-5c9ca4b13dc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:09:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "27019" uid: 45f00b94-4f6e-4c77-b0d1-7acbf718c59e kind: Event lastTimestamp: "2026-07-08T17:10:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:10:23Z" name: error-404-isvc-ab489-predictor.18c05fa01eded907 namespace: kserve-ci-e2e-test resourceVersion: "27834" uid: 239c0044-f9d3-4081-9893-9a84acf5f699 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:09:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "27019" uid: 45f00b94-4f6e-4c77-b0d1-7acbf718c59e kind: Event lastTimestamp: "2026-07-08T17:10:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:10:23Z" name: error-404-isvc-ab489-predictor.18c05fa02009cbea namespace: kserve-ci-e2e-test resourceVersion: "27835" uid: d7e4fe5b-573e-4737-838c-51249ccab60d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: ed7cdd5c-61ef-4961-97c5-6c5fcc25bec2 kind: Event lastTimestamp: "2026-07-08T17:09:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:35Z" name: error-404-isvc-ab489.18c05f8ea4209730 namespace: kserve-ci-e2e-test resourceVersion: "27419" uid: 041daf12-e0fc-4276-836d-01e272b5d327 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: ed7cdd5c-61ef-4961-97c5-6c5fcc25bec2 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'Failed to update status for InferenceService "error-404-isvc-ab489": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ab489": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-ab489.18c05f8ea4abdc2f namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: c3769f60-ffdc-4cd4-a8e0-7625011ae776 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: ed7cdd5c-61ef-4961-97c5-6c5fcc25bec2 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ab489": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:38Z" name: error-404-isvc-ab489.18c05f8ea4ac3415 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 46508294-2c6a-4182-9f4f-65a935736bb3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "27414" uid: ed7cdd5c-61ef-4961-97c5-6c5fcc25bec2 kind: Event lastTimestamp: "2026-07-08T17:09:35Z" message: InferenceService [error-404-isvc-ab489] is Ready metadata: creationTimestamp: "2026-07-08T17:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:35Z" name: error-404-isvc-ab489.18c05f9bdcf7219e namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: 77b54f80-4afd-4bd0-90e8-48492d5b295b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:59:13.347349Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b0389c5e5 namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: d7756677-62a4-46a8-a5c1-6ef7f640e48d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b1cb6cfd4 namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: b0ae355d-2860-4ed4-b580-a067f4a159a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b1dffe43c namespace: kserve-ci-e2e-test resourceVersion: "22046" uid: 22d3f946-f503-42b9-95d1-5ef6439d565a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b20e3e314 namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 77e1c991-b761-4fb3-9279-0d15602b7071 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b215dfbb0 namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: 5871318b-2d61-486d-8d9d-b7f31d1e93d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b216a3d97 namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: caa89b12-6a24-47cd-b7e3-f96f1e288f8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b2424ea41 namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: 4dde7e66-79fa-4a53-9307-497d587956a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b24921321 namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: 6d888709-fa85-445d-9072-44e95b6b436a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T16:59:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:51Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f0b9be12587 namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 9662ca4e-5bed-4a12-bb4c-af2abff5a6b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f8c282c23fb namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: f56a95c1-213b-477b-9fff-41a8c5495432 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 099312d7-f64f-48a3-801e-18d3e9055d25 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79.18c05f8c282c9dda namespace: kserve-ci-e2e-test resourceVersion: "26544" uid: 0ec57b31-6d7f-4d67-b7d2-2acc53eef023 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2f8c-predictor-7c69fd84bb namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: 2325f1ff-977f-4967-8348-90c94762cddb kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Created pod: error-404-isvc-b2f8c-predictor-7c69fd84bb-cxh79' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor-7c69fd84bb.18c05f0b02f5eda8 namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: 9a59337d-2f3a-4b75-80e4-debcfd2fc4ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: f63659d4-860a-40f3-b40a-0e5904880a8c kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Scaled up replica set error-404-isvc-b2f8c-predictor-7c69fd84bb from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c-predictor.18c05f0b01556ec2 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: 1a46a143-f1a2-4345-9863-6408732e52f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 95da8410-543d-457b-bd8f-4c622536a20e kind: Event lastTimestamp: "2026-07-08T16:59:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:43Z" name: error-404-isvc-b2f8c-predictor.18c05f0e7fed1138 namespace: kserve-ci-e2e-test resourceVersion: "22319" uid: 2e9dbaba-80fa-4569-99fc-44472926d663 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 95da8410-543d-457b-bd8f-4c622536a20e kind: Event lastTimestamp: "2026-07-08T16:59:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:43Z" name: error-404-isvc-b2f8c-predictor.18c05f0e806c03ad namespace: kserve-ci-e2e-test resourceVersion: "22320" uid: efade94c-ecc9-4f87-9378-c8088f59300b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: 95da8410-543d-457b-bd8f-4c622536a20e kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:13Z" name: error-404-isvc-b2f8c-predictor.18c05f157ddfe2f4 namespace: kserve-ci-e2e-test resourceVersion: "22578" uid: 48553d9c-de6c-4d41-bd2c-fd7d4ad96d5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: 95da8410-543d-457b-bd8f-4c622536a20e kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:13Z" name: error-404-isvc-b2f8c-predictor.18c05f157e591fba namespace: kserve-ci-e2e-test resourceVersion: "22579" uid: 323014dc-e735-4361-b45b-91b00e13e8ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: f8c96e29-48a1-4f49-8ea2-2db878633943 kind: Event lastTimestamp: "2026-07-08T17:00:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:00:02Z" name: error-404-isvc-b2f8c.18c05f0b0155838c namespace: kserve-ci-e2e-test resourceVersion: "22478" uid: 77c2a566-1771-4ad2-ac90-d07afa6a1013 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: f8c96e29-48a1-4f49-8ea2-2db878633943 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-b2f8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2f8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c.18c05f0b01b82e6d namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: d982d566-9f9d-4461-8d14-69ba2ae799e6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: f8c96e29-48a1-4f49-8ea2-2db878633943 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2f8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: error-404-isvc-b2f8c.18c05f0b01b8798a namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 72fa198e-f5ca-4865-926f-655ccb8948e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: f8c96e29-48a1-4f49-8ea2-2db878633943 kind: Event lastTimestamp: "2026-07-08T17:00:02Z" message: InferenceService [error-404-isvc-b2f8c] is Ready metadata: creationTimestamp: "2026-07-08T17:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:00:02Z" name: error-404-isvc-b2f8c.18c05f1657774542 namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: be690c48-bad4-4ed2-8ef3-ee088158785c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:08:28.117931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d09b9-predictor-6cb94d46cb-489qc to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c2e71102f namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 2a8dc15e-0bf0-4e8a-8bb2-50a7f50b90bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c48c00c8b namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: e6e137a4-6631-41ab-a951-4af6944847ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c4a5357ad namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 89fae576-1970-4899-a6f8-299b8e4a43e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c4ce994f2 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: cb58ee60-52e1-4a37-b0ab-0b90391519d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c4d53adf2 namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: de307d6d-7fa4-41f8-bf6a-e0e078e4d950 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c4d5d1b7f namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: ba551b10-3da6-4f58-9c91-b8115885fcff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c50538ccd namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 6cbb5c2a-24f6-4dc9-8692-81fe6fcffd9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c50cc8b48 namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: b90f1dbe-87a3-4aae-b690-85063cc6264b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:09:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:05Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c05f8c74d0365c namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: 53fcb01d-50c8-40a1-b4cf-6c92635be56a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:25:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:47Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c0607e3839e149 namespace: kserve-ci-e2e-test resourceVersion: "34256" uid: b6eed55f-db06-408a-a8a4-8c57ced4aafe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:25:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:47Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c0607e383a6b32 namespace: kserve-ci-e2e-test resourceVersion: "34257" uid: 425bae79-e7ed-49ea-bf7f-94f0e552c744 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 86378337-6f02-4b24-b099-66c502100f55 kind: Event lastTimestamp: "2026-07-08T17:25:50Z" message: 'Readiness probe failed: Get "https://10.133.0.46:8643/healthz": dial tcp 10.133.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:50Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc.18c0607ed5836960 namespace: kserve-ci-e2e-test resourceVersion: "34319" uid: cec2ea1f-a172-4c49-9f3b-d0b30a5e62a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d09b9-predictor-6cb94d46cb namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 18c39dbc-4a6a-4292-b680-36d237ae7c38 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created pod: error-404-isvc-d09b9-predictor-6cb94d46cb-489qc' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor-6cb94d46cb.18c05f8c2de69943 namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: 1a97f61d-0f3c-4ce7-8302-d96d35a9fc17 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: dc80d285-6a3d-4acf-8c61-343f1d7d4629 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Scaled up replica set error-404-isvc-d09b9-predictor-6cb94d46cb from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9-predictor.18c05f8c2c60284b namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 6a47fd82-ce1c-437b-9cc7-c769a7ccaeea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: da4f872c-dea8-4256-be05-342b45b27a9a kind: Event lastTimestamp: "2026-07-08T17:08:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:58Z" name: error-404-isvc-d09b9-predictor.18c05f8faae6ba4b namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: 1fda7511-61f5-4322-be03-4fae36c1ccc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: da4f872c-dea8-4256-be05-342b45b27a9a kind: Event lastTimestamp: "2026-07-08T17:08:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:58Z" name: error-404-isvc-d09b9-predictor.18c05f8fab592879 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: e19d1f75-70bb-4149-8d78-9d650bae8a9b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:09:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: da4f872c-dea8-4256-be05-342b45b27a9a kind: Event lastTimestamp: "2026-07-08T17:09:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:43Z" name: error-404-isvc-d09b9-predictor.18c05f96a993b5af namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: c7e2a1a1-4cf3-4968-95e6-500cfab31614 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:09:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: da4f872c-dea8-4256-be05-342b45b27a9a kind: Event lastTimestamp: "2026-07-08T17:09:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:43Z" name: error-404-isvc-d09b9-predictor.18c05f96a9f4287a namespace: kserve-ci-e2e-test resourceVersion: "27491" uid: 9c991dac-2180-4898-9495-5268ed389920 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: 63e686c2-f239-484f-b918-4abe59247b02 kind: Event lastTimestamp: "2026-07-08T17:09:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:15Z" name: error-404-isvc-d09b9.18c05f8c2c63ff07 namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: ba8def5b-9e66-4a9d-8d1e-b9477e63b108 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: 63e686c2-f239-484f-b918-4abe59247b02 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-d09b9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d09b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9.18c05f8c2cfd351d namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 22f1ebae-b5df-4005-99f2-b530e850026f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: 63e686c2-f239-484f-b918-4abe59247b02 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d09b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:28Z" name: error-404-isvc-d09b9.18c05f8c2cfd79df namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: 043d1cdd-fc1b-4d08-895f-3de1b150af4c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 63e686c2-f239-484f-b918-4abe59247b02 kind: Event lastTimestamp: "2026-07-08T17:09:15Z" message: InferenceService [error-404-isvc-d09b9] is Ready metadata: creationTimestamp: "2026-07-08T17:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:15Z" name: error-404-isvc-d09b9.18c05f9733c13842 namespace: kserve-ci-e2e-test resourceVersion: "27220" uid: b910f0ea-e3dc-4bc7-bf01-74a5cb6a0638 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:18.298686Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:18Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e568f09b63e namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 44da84f1-607d-4c1f-bc30-b58e6fd24730 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:18Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e56a85d5f89 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: ee28b07c-a85a-4696-a7dc-cf13f996eb2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e56a99e26e5 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 2abfd8fc-6214-45bb-a5fd-4bd3ede2573f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" in 15.714s (15.714s including waiting). Image size: 1334693979 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5a524649c9 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 6e6b22bc-9077-4bd6-9c79-a6849455b2bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5a56a77095 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: ce569a5e-3953-42ad-b790-8df1b282c42b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5a5717adb5 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 03cf7593-af63-494a-9ebd-29c7f290569c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5a5723b4bc namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: 49e4e7c5-da53-4905-9396-f4476c0235f8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.528s (2.528s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5aede162a9 namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 2d9415c9-ec49-4810-be39-0bade8f5922c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5af1776131 namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: cfaf6a68-abf7-460c-9838-6eb65a0c1206 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5af2005aa1 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 3f86738f-744e-45b4-9dd2-7be99dc3a79f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:47:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:13Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e5b0c5d2d74 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: e9ba7196-0596-4dc0-8e18-1be8f7176a35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e6b7a38b30d namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 4c9ed472-cff3-45fe-b254-6f7515606075 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e6b7a391c52 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: ec2906d7-2553-4e1a-80db-f66a87f1b3ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 356dfcc5-b1f8-4b29-ac1b-d9525b41c2b8 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7.18c05e6b94916784 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: 8e1312a2-acb9-47de-9d8f-cf50ef9cf46d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f6c4d-predictor-68f94b898b namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 9d617336-c1c3-4544-a576-be2faf8444d0 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created pod: error-404-isvc-f6c4d-predictor-68f94b898b-49jc7' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: error-404-isvc-f6c4d-predictor-68f94b898b.18c05e568e6efa62 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 43ae900a-28dc-41bc-a843-954fa15ef41c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 5e58b9e5-5bd2-4bc5-a228-d83522ae5346 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Scaled up replica set error-404-isvc-f6c4d-predictor-68f94b898b from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: error-404-isvc-f6c4d-predictor.18c05e568ba0f881 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 2a96a239-a919-4cef-bb2d-deb5d0f9f5ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: f285eaaf-4dc1-44d7-bf83-5ee492699c64 kind: Event lastTimestamp: "2026-07-08T16:46:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:48Z" name: error-404-isvc-f6c4d-predictor.18c05e5a0a40420c namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 2b519e2c-f2c6-4fa7-910f-9eeec40c8cd0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: f285eaaf-4dc1-44d7-bf83-5ee492699c64 kind: Event lastTimestamp: "2026-07-08T16:46:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:48Z" name: error-404-isvc-f6c4d-predictor.18c05e5a0aaf4973 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: bf492a6b-baac-480c-9445-bf9b157904e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:47:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: f285eaaf-4dc1-44d7-bf83-5ee492699c64 kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: error-404-isvc-f6c4d-predictor.18c05e610844c189 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: f00d217c-6d67-4caa-8284-c3ecb472ad49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:47:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: f285eaaf-4dc1-44d7-bf83-5ee492699c64 kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: error-404-isvc-f6c4d-predictor.18c05e6108d9aff3 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: ffeb0cbb-f6b4-43fc-8bd9-de98739e4d14 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6c4d namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 842c2d0d-bf90-40ee-bd28-eb9269c84fc9 kind: Event lastTimestamp: "2026-07-08T16:47:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:23Z" name: error-404-isvc-f6c4d.18c05e568bd1a408 namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: bfbf33ff-0bc1-48ba-9928-e1d4782e2403 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6c4d namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 842c2d0d-bf90-40ee-bd28-eb9269c84fc9 kind: Event lastTimestamp: "2026-07-08T16:47:23Z" message: InferenceService [error-404-isvc-f6c4d] is Ready metadata: creationTimestamp: "2026-07-08T16:47:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:23Z" name: error-404-isvc-f6c4d.18c05e65cbadacfe namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 045a268d-24f4-46cb-8dd1-62bae6835432 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:48:48.199998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e7975da3c26 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 3d47104c-b733-4773-a08e-b4afed9f33da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e798fa1adb5 namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: cbe1ff93-1085-4754-b655-0263e58b0558 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e7990eb93e5 namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: c0af5204-71e3-4361-9253-a4aa63152282 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e7993a319c6 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 63f8c269-4912-46d3-91ff-f8665a0c1c46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e7994001876 namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 74278373-cdb1-4b6c-870f-61530eab54ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e799408fd78 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: caa74b57-9d1e-4d0c-b426-e3eac5a811b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e79988c1e5f namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 1d81dc36-7f68-46d0-bfa4-cb54b1c0f1cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e7998f34f7b namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 7ffe7af1-c0ce-40ff-a8c0-11e3f69552d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:48:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:58:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:48:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:05Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05e79acfebca3 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: e2ea228c-3555-4f0d-bf4b-c1bbe0dae061 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:02Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05efaa0196fc6 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: b9acfb62-4f79-4c33-b31f-c4e70fc8ea55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:02Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05efaa019d632 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: bdb86002-6a9e-4b23-9658-9f0a16531836 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 660316b3-0555-4e3f-a196-c24c318c42c9 kind: Event lastTimestamp: "2026-07-08T16:58:05Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:05Z" name: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b.18c05efb2168c6de namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 17a39fe6-8069-4afc-bceb-9bce0761276b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fcbfa-predictor-694c77d57b namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 60b4741d-2dcb-4da9-9069-b39740f19da1 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created pod: error-404-isvc-fcbfa-predictor-694c77d57b-zxw5b' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor-694c77d57b.18c05e7975456308 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 2fe8d28c-cf0a-46bf-bdde-3e330658fb90 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 0bb1e964-6ce8-472d-a351-8bf6146167d8 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Scaled up replica set error-404-isvc-fcbfa-predictor-694c77d57b from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: error-404-isvc-fcbfa-predictor.18c05e79737a5c3b namespace: kserve-ci-e2e-test resourceVersion: "16725" uid: 2d6cda71-f730-40f1-b397-f829fea0ea44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 2ccb817f-c3ba-4e0a-8135-85d725a8b514 kind: Event lastTimestamp: "2026-07-08T16:49:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:18Z" name: error-404-isvc-fcbfa-predictor.18c05e7cf2298718 namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: c8200b66-b5de-4742-ab7d-6ee483f3f7d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 2ccb817f-c3ba-4e0a-8135-85d725a8b514 kind: Event lastTimestamp: "2026-07-08T16:49:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:18Z" name: error-404-isvc-fcbfa-predictor.18c05e7cf281cbbf namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: a2dd82f0-b731-41ac-8563-b54af8be8f60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:49:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 2ccb817f-c3ba-4e0a-8135-85d725a8b514 kind: Event lastTimestamp: "2026-07-08T16:50:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:03Z" name: error-404-isvc-fcbfa-predictor.18c05e83f0337345 namespace: kserve-ci-e2e-test resourceVersion: "17455" uid: 589264bf-6987-4bb9-9450-9bdbbf5a3851 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:49:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 2ccb817f-c3ba-4e0a-8135-85d725a8b514 kind: Event lastTimestamp: "2026-07-08T16:50:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:03Z" name: error-404-isvc-fcbfa-predictor.18c05e83f0b03d2a namespace: kserve-ci-e2e-test resourceVersion: "17456" uid: 23bd354a-43fc-4d4b-ac7b-ff5225ff9a83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 3ae741f0-ce82-4e3f-9993-a38f4fec510d kind: Event lastTimestamp: "2026-07-08T16:49:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:35Z" name: error-404-isvc-fcbfa.18c05e7973b17d7f namespace: kserve-ci-e2e-test resourceVersion: "17208" uid: 76f207f2-c611-4e28-9fd1-a428460c7204 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: 3ae741f0-ce82-4e3f-9993-a38f4fec510d kind: Event lastTimestamp: "2026-07-08T16:49:35Z" message: InferenceService [error-404-isvc-fcbfa] is Ready metadata: creationTimestamp: "2026-07-08T16:49:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:35Z" name: error-404-isvc-fcbfa.18c05e8467be61c4 namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: 71455a16-827b-400e-9022-747e00cd93ec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:17.905254Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t to ip-10-0-130-254.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:17Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5677967492 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: d5c787de-1224-4acb-9c78-82de7a95e1a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5692b4aed0 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 69311e54-6e75-41f8-8db0-36b8c69399d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e569450d390 namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 10fad71c-5778-49ac-b30a-4d30ad6c771b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5697ec5f55 namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 42829892-ad4e-483d-978f-30766644b01d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e56986cc10d namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: d7503f1a-91d3-4b68-91c5-03d79e3c83ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:22Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" metadata: creationTimestamp: "2026-07-08T16:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:22Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e57a466f5b9 namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: bebbe1da-b28c-451a-8e9b-7b0eb14605a3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" in 29.724s (29.724s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5e901dcff8 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 6aae9a6e-f94f-485b-b0c9-8bab5d0a4404 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5e961acf37 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 7d3200c8-0633-4b75-8475-c0d2382e9fc5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5e9710ae5b namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 4425d264-82af-4d31-910d-6f5bcb074ecd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5e97254609 namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: af16f88e-ebc7-4a52-aea0-1ff964ceea03 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.16s (2.16s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:54Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5f17e6b949 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: eb185248-d49e-4b5c-b232-f0f2cc7133b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:55Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5f1d27cde5 namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 3542f853-44d5-46f3-9656-e48c38f43b99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:46:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:55Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5f1da7b2ad namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: c6a854dd-859d-4ae1-b60f-2a66e8470958 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:48:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:51Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e5f267ebdfb namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: d199a284-928f-48bd-bdbd-d04e4bfb8e80 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e796f1ab203 namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: f3b9ebb4-5651-4b0d-a51c-dca6559a8cdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e796f1b3846 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: f1380e62-522b-4d73-8d89-bcb1eb50bd96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: cf187438-a689-45c2-8fba-bdc213938cda kind: Event lastTimestamp: "2026-07-08T16:48:51Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:51Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t.18c05e7a290d8764 namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: b5137ccb-0677-4198-a1f8-1df3af17912f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-f669c5c7f namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: 02bd4224-3b9a-4b67-a85d-8e909cc7bd98 kind: Event lastTimestamp: "2026-07-08T16:46:17Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-f669c5c7f-qm88t' metadata: creationTimestamp: "2026-07-08T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:17Z" name: isvc-sklearn-graph-1-predictor-f669c5c7f.18c05e567704f019 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 0574367d-6f65-43df-889a-37c9c537b52f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 8718ad8c-2da3-4b1b-bde5-ca8f26781a77 kind: Event lastTimestamp: "2026-07-08T16:46:17Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-f669c5c7f from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:17Z" name: isvc-sklearn-graph-1-predictor.18c05e566f32b456 namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 8abf9d26-4516-4f34-bacc-ef8f2f7aaf26 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:46:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: af053d5b-b81c-4e5a-8ac6-5bc92460afb4 kind: Event lastTimestamp: "2026-07-08T16:47:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:32Z" name: isvc-sklearn-graph-1-predictor.18c05e59f4c79ad0 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 61d70bff-73ad-4d35-b978-b47b995a4cb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:46:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: af053d5b-b81c-4e5a-8ac6-5bc92460afb4 kind: Event lastTimestamp: "2026-07-08T16:47:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:32Z" name: isvc-sklearn-graph-1-predictor.18c05e59f52e26fa namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 5e83454a-15c1-404c-8097-1da9e05390db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:47:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: af053d5b-b81c-4e5a-8ac6-5bc92460afb4 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: isvc-sklearn-graph-1-predictor.18c05e6b70487937 namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 991659b0-24b3-4685-9ac8-82ef4d31e792 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:47:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: af053d5b-b81c-4e5a-8ac6-5bc92460afb4 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: isvc-sklearn-graph-1-predictor.18c05e6b70a8da12 namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: ccfadc70-c628-4140-b887-6c6b58e44a29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:46:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 096030f2-b509-406a-a803-8254fd5ca70d kind: Event lastTimestamp: "2026-07-08T16:48:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:11Z" name: isvc-sklearn-graph-1.18c05e56756e3920 namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 30516d8d-2bfb-4c50-ba34-abb95f284c59 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 096030f2-b509-406a-a803-8254fd5ca70d kind: Event lastTimestamp: "2026-07-08T16:48:11Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-08T16:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:11Z" name: isvc-sklearn-graph-1.18c05e70df4125d7 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 6a8176bf-ad68-4703-b60c-079ed947383b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:18.408193Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 to ip-10-0-130-254.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e569590b995 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: f21157e5-86ed-43ab-8549-065150b2a622 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e56b131f5dc namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 210f71b8-464f-49e9-bce7-410de889bde8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e56b2bef707 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 4129301e-c1e5-44b3-95f1-b65002863583 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e56b6513d94 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 72f27e27-9fe6-4a24-9c0a-aef8e6062808 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e56b6e95858 namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 8ee1c0d1-ed44-4377-ac09-9c6e61aa538d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:22Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" metadata: creationTimestamp: "2026-07-08T16:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:22Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e57a44e8912 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 741065ec-7185-4762-b478-4df844c1a542 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:51Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" in 28.841s (28.841s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:51Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5e5b685cf4 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: dbc3eb72-8fbc-4f48-9793-0ccb6ed3982b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:51Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5e61920179 namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 63b9f6c0-4961-49da-866c-f79472a0372d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:51Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5e633e5e76 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 279f9cb5-c627-4761-a0b7-b64cbafa7b51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:51Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5e6351b103 namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: e185dadb-3f04-4f21-b404-8253a2b58c54 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.042s (3.042s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:54Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5f18b00371 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: f17b7827-7705-400f-a3ba-510e1da4499f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:55Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5f1d26b4a1 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 0b7bff5b-a6a9-41a8-9cf2-291b0873b7ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:46:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:55Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5f1db4d0e3 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 57ae20c9-f549-4270-86fa-284946cd4fd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:48:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:51Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e5f265d4543 namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: 6de9fd97-e271-4284-82bf-0a1e859cb277 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e796ac1c884 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 1ce91c2e-41d1-4e4a-8712-47f8ee1c972f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e796ac2fe61 namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 070b1908-7085-40eb-958e-2769b7cac894 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 0b0a9047-2857-4191-b645-b3297003c8a7 kind: Event lastTimestamp: "2026-07-08T16:48:51Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:51Z" name: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8.18c05e7a290cf8b5 namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 763c2121-dab1-47a5-b2b4-a58bafd0b847 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7dd9885465 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: 67392c54-f5b1-41f3-9c0b-7299725fac57 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7dd9885465-zlgg8' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor-7dd9885465.18c05e56951e5a32 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: a81045a2-045f-40c3-840a-d89aa9475717 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: 7513d1a6-b574-485a-8e96-b119bb555ea4 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7dd9885465 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: isvc-sklearn-graph-2-predictor.18c05e569257805d namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 99c45d5c-e647-49a7-aa0e-72f7940adcf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 795e5f7f-26b3-4a95-9a8e-69f9717c46d5 kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: isvc-sklearn-graph-2-predictor.18c05e5a10ed8390 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: c415f35c-63ad-495f-9049-2ee97daf01cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 795e5f7f-26b3-4a95-9a8e-69f9717c46d5 kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: isvc-sklearn-graph-2-predictor.18c05e5a11683028 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 2f6c7310-4270-4d8b-8286-b83135a46f6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 795e5f7f-26b3-4a95-9a8e-69f9717c46d5 kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:33Z" name: isvc-sklearn-graph-2-predictor.18c05e6b8b5ad80f namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 6d9f3080-c423-4781-ae71-ce5855510485 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 795e5f7f-26b3-4a95-9a8e-69f9717c46d5 kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:33Z" name: isvc-sklearn-graph-2-predictor.18c05e6b8bc3ee20 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 8579620d-4df2-4ee9-9385-0c9a1c15d398 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 16c862e5-5455-44c4-aa7d-742369f2bbb0 kind: Event lastTimestamp: "2026-07-08T16:48:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:11Z" name: isvc-sklearn-graph-2.18c05e56925ca14f namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 809528c2-2c25-4f8c-9fc2-4119e3ef1ce3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 16c862e5-5455-44c4-aa7d-742369f2bbb0 kind: Event lastTimestamp: "2026-07-08T16:48:11Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-08T16:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:11Z" name: isvc-sklearn-graph-2.18c05e70e1e86623 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: db12ea8d-c417-4660-a195-a9bd6a15320a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:18.202155Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-h82kc to ip-10-0-130-254.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e568948d2dd namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: f53d6e83-8ed0-4877-8500-6ea33c8e019e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e56a4f17019 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: ddb8d40b-543e-4663-a60e-cd0b036c69b1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e56a658ca96 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: f0dfa244-2fde-4c55-9e0f-b3f24d3e99a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e56a9235c1e namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 4acf9f84-ec10-4518-bc19-312860460ab4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e56a9b4ac75 namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: bae1e0c5-090a-4604-aac0-d7ffe1d77d78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:22Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-08T16:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e57a482e26c namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 54c6dc82-06d9-4033-9b25-ba8ea0e2f04a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.249s (29.249s including waiting). Image size: 1305547515 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5e73e90cea namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: 797eff70-00af-4d2b-bf5e-d6fa5af06fcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5e7adfe194 namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 26bcdd78-99a0-42d7-b58a-b2c4c9f72870 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5e7c86e478 namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: e110a758-547a-4601-a05e-04a80c7d5cb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5e7c96d8ed namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: e059f2ad-607c-47e7-a4f8-0ecf692e6f37 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.605s (2.605s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5f17eb73df namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: e836cd42-500d-409b-a7f6-788ec2ca69ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5f1d1aab7f namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: bbd448c7-5cb0-44fa-af52-027d39c0dec8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:46:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5f1db4ad07 namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: cc5a7f4d-be94-4ccf-a0bb-152d851521fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T16:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:47:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e5f6287c94a namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 44ee0b69-3a33-4f85-84c5-a5c9c4e28e7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e797218b2ff namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 53156f59-efb8-4709-b03c-5408aa8204f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e79721949f9 namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 7672ba38-3f61-4414-9f6c-c3666c602433 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:48:53Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-08T16:48:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:53Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e7aa0ce00ce namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: c3c889dc-f7e0-4236-bd99-0164fae0fe77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-h82kc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: fe3de155-abb6-455d-99cb-99ed3a204f6d kind: Event lastTimestamp: "2026-07-08T16:48:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: i/o timeout' metadata: creationTimestamp: "2026-07-08T16:48:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:53Z" name: isvc-xgboost-graph-predictor-555fc69d56-h82kc.18c05e7aa11faf42 namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: a271fb4c-8381-413a-8e0f-e5f95899ccbb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-254.ec2.internal source: component: kubelet host: ip-10-0-130-254.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 91a4827c-9e6a-4bcb-9636-b415e323ee20 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-h82kc' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c05e5688e19c3c namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 82103cf0-afe2-4a51-aa3d-52fe945f2b54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: d50199fb-1198-413b-98c3-3814ae6b9d14 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: isvc-xgboost-graph-predictor.18c05e568611636c namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: ea60e718-cd1e-49f5-bbbe-9c6226e25b32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 16be7b2f-3cd2-41b0-80fe-d769eb70e9be kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: isvc-xgboost-graph-predictor.18c05e5a04a2fb3d namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 55b20e69-5ccb-4076-a4a0-77e755720df7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 16be7b2f-3cd2-41b0-80fe-d769eb70e9be kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: isvc-xgboost-graph-predictor.18c05e5a05066526 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: a1e04dbd-04c1-4542-afec-a9df3da8170c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 16be7b2f-3cd2-41b0-80fe-d769eb70e9be kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:33Z" name: isvc-xgboost-graph-predictor.18c05e64815d0a2a namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 02525a42-3c91-405c-8c5d-4e6358d6e114 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 16be7b2f-3cd2-41b0-80fe-d769eb70e9be kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:33Z" name: isvc-xgboost-graph-predictor.18c05e6481cad494 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: c15d4efd-a06e-45ff-98ca-8479bc3f6ba2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 6cd34866-b697-4993-909b-b77d6b860819 kind: Event lastTimestamp: "2026-07-08T16:48:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:02Z" name: isvc-xgboost-graph.18c05e568635cec2 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 9ce28962-9dd6-462b-959f-73ff0734b1e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 6cd34866-b697-4993-909b-b77d6b860819 kind: Event lastTimestamp: "2026-07-08T16:48:02Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-08T16:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:02Z" name: isvc-xgboost-graph.18c05e6ec794d167 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 33affac6-a6ca-478a-ba59-07ada3b8ed53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:48:37.793387Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-658ff57b89-v7pjz to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:48:37Z" name: model-chainer-658ff57b89-v7pjz.18c05e770991e978 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: ce5093e5-0bf3-4ff8-b51f-486603fe9c61 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: model-chainer-658ff57b89-v7pjz.18c05e77150bb83e namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: a2a21b95-5694-470d-9ac4-081d33a25670 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:48:38Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:48:38Z" name: model-chainer-658ff57b89-v7pjz.18c05e7746c56797 namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: c0d592a4-4ee1-4692-b4a6-433781445692 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:48:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:38Z" name: model-chainer-658ff57b89-v7pjz.18c05e7748070b04 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 562942ad-ebad-4c24-bc49-9c23cbd50a08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:48:38Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-08T16:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:38Z" name: model-chainer-658ff57b89-v7pjz.18c05e774b10f268 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 301116b9-1c69-4e3c-be0e-7b034a5fdc63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:48:38Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-08T16:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:38Z" name: model-chainer-658ff57b89-v7pjz.18c05e774b8aa147 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: de250fef-4018-46dc-bb85-065867a88290 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:48:47Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-08T16:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:47Z" name: model-chainer-658ff57b89-v7pjz.18c05e7963a58d8b namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 6bc91068-065a-4a9a-a485-afbacb50f6cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-658ff57b89-v7pjz namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 66e72dfb-990c-402c-bc90-8a94b59ec1f1 kind: Event lastTimestamp: "2026-07-08T16:49:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:15Z" name: model-chainer-658ff57b89-v7pjz.18c05e79ef68ac8f namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 61dc7b37-91e8-4e10-8d72-9b992ed60127 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-658ff57b89 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 681ef8c5-502e-4d27-a46f-712f83e62f91 kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: 'Created pod: model-chainer-658ff57b89-v7pjz' metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:37Z" name: model-chainer-658ff57b89.18c05e7709234c21 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 96a7f4e4-6e0f-43cc-95c9-11339896ecc7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: e362e2e9-1f4f-457e-897d-837284e298a6 kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Scaled up replica set model-chainer-658ff57b89 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:37Z" name: model-chainer.18c05e770869b0c1 namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: ce4a9ddc-1efe-4f2f-8de7-260ccd7f5805 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 93458fbd-3127-4a99-8944-493dbd49f0f1 kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:37Z" name: model-chainer.18c05e770e171d29 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 1de11d5b-6558-4fe1-8b48-505471b6c601 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:59:02.997980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-178bc-84fb875cf-fkbbh to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:59:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:59:02Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f089aab0c89 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: f50de380-1f6f-4eaa-91d0-0845a0b23fbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:59:03Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f08b4b67192 namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 4d1601f4-e5f3-4b2d-a3f3-dd5af55ea335 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-178bc} kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:03Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f08b6064df1 namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: dd6d23e0-2300-455e-8be7-e349c9203d38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-178bc} kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: 'Created container: sequence-graph-178bc' metadata: creationTimestamp: "2026-07-08T16:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:03Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f08b8c02ec4 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: 1d462a03-071a-4ece-b521-d5e2b6754f99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-178bc} kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: Started container sequence-graph-178bc metadata: creationTimestamp: "2026-07-08T16:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:03Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f08b93a05a7 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: f84beb39-f8eb-450a-89f8-02df24fd8667 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-178bc} kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Stopping container sequence-graph-178bc metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f0af34c766b namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: 86fc270d-bbe2-4bb5-ade2-25a40c7404f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-178bc} kind: Pod name: sequence-graph-178bc-84fb875cf-fkbbh namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 44269f71-20e0-4978-8a33-1ba022775385 kind: Event lastTimestamp: "2026-07-08T16:59:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:39Z" name: sequence-graph-178bc-84fb875cf-fkbbh.18c05f0b530841f0 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: a63c6529-ac7b-4966-870a-c3838a89d7f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-178bc-84fb875cf namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: c7764cd3-5312-443c-89f7-cf69a80d9a77 kind: Event lastTimestamp: "2026-07-08T16:59:02Z" message: 'Created pod: sequence-graph-178bc-84fb875cf-fkbbh' metadata: creationTimestamp: "2026-07-08T16:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:02Z" name: sequence-graph-178bc-84fb875cf.18c05f0899f92ad8 namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 244134a3-8ace-4647-8add-3b101a3ed5dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-178bc namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 1c27812f-2767-4fbc-a4b4-9b7f43e339b5 kind: Event lastTimestamp: "2026-07-08T16:59:02Z" message: Scaled up replica set sequence-graph-178bc-84fb875cf from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:02Z" name: sequence-graph-178bc.18c05f0898e660a7 namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 56d7d1be-3c3e-41c6-9b1a-e87a9b4c34b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-178bc namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 8e5d318b-fd7f-423d-9eb1-342e9a15c1ec kind: Event lastTimestamp: "2026-07-08T16:59:03Z" message: InferenceGraph [sequence-graph-178bc] is Ready metadata: creationTimestamp: "2026-07-08T16:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:03Z" name: sequence-graph-178bc.18c05f089ee5051e namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: 0d0d91df-b2ff-40dd-83bd-d9275fa64e4c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:00:13.329127Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b2f8c-754875c9d4-hlsk8 to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f18fabb1be9 namespace: kserve-ci-e2e-test resourceVersion: "22573" uid: 0185e495-501f-4c67-9599-0cedb870961d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22576" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f19144d20e2 namespace: kserve-ci-e2e-test resourceVersion: "22591" uid: e74a0a6f-6aa1-4812-b57b-1c0f0dada2a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b2f8c} kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f191593fcb3 namespace: kserve-ci-e2e-test resourceVersion: "22594" uid: 1b06e46d-cb86-4eab-903c-da2a2e452a98 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b2f8c} kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: 'Created container: sequence-graph-b2f8c' metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f19188df7df namespace: kserve-ci-e2e-test resourceVersion: "22595" uid: da558372-3c73-4bc4-9855-e83fcdffce69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b2f8c} kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: Started container sequence-graph-b2f8c metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f1919016dc7 namespace: kserve-ci-e2e-test resourceVersion: "22596" uid: 20d5be48-3abc-4967-8f32-532aba10f80b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b2f8c} kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: "2026-07-08T17:08:27Z" message: Stopping container sequence-graph-b2f8c metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:27Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f8c1e2aafaa namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: be37e7dc-a58d-4557-83eb-7ffc853bc64f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b2f8c} kind: Pod name: sequence-graph-b2f8c-754875c9d4-hlsk8 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: 5eab1a8c-0388-4847-97a7-6edf3207871d kind: Event lastTimestamp: "2026-07-08T17:08:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:54Z" name: sequence-graph-b2f8c-754875c9d4-hlsk8.18c05f8c9c6d1936 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: a0ec3bee-2ac2-4618-93a1-c78a376c93e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b2f8c-754875c9d4 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: d2a83e0b-85a7-4410-85aa-24b4d6edb8f5 kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: 'Created pod: sequence-graph-b2f8c-754875c9d4-hlsk8' metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c-754875c9d4.18c05f18fa8991cc namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: ae9a51c7-1fa8-4ad8-882d-c0d008bf3489 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: 2c15ada1-6f40-4838-b2f5-5b9abe5ff83a kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: Scaled up replica set sequence-graph-b2f8c-754875c9d4 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c.18c05f18f96c8f8a namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: 8dfd5a99-cc19-4866-bc6e-54a16a533028 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "22581" uid: 2f6a84af-6ae1-4d97-a98b-4bb35d81155e kind: Event lastTimestamp: "2026-07-08T17:00:13Z" message: InferenceGraph [sequence-graph-b2f8c] is Ready metadata: creationTimestamp: "2026-07-08T17:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:00:13Z" name: sequence-graph-b2f8c.18c05f18ff14fe45 namespace: kserve-ci-e2e-test resourceVersion: "22582" uid: 48cd1ca7-07a8-426b-979c-5a788859c9cf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:00:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: 13efdf33-1633-4c68-851b-7954c903b57e kind: Event lastTimestamp: "2026-07-08T17:00:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:58Z" name: sequence-graph-b2f8c.18c05f1c78da5177 namespace: kserve-ci-e2e-test resourceVersion: "22916" uid: 4252dcae-101a-47c5-aead-9ba4440ed855 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:00:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: 13efdf33-1633-4c68-851b-7954c903b57e kind: Event lastTimestamp: "2026-07-08T17:00:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:58Z" name: sequence-graph-b2f8c.18c05f1c793d0e49 namespace: kserve-ci-e2e-test resourceVersion: "22917" uid: 59218df1-2356-42fc-89c6-bc0f0d5e9727 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:49:48.116983Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fcbfa-6dfb4df4f4-47wtl to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05e87692ed9f4 namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 2c85d724-2039-455f-8ddc-e73cd480c666 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05e8782c0332b namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 991fc03d-467c-4939-98eb-f1f47dbef224 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fcbfa} kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05e8783fb661e namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: acb70cb3-1be3-4a23-b463-27f465a33151 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fcbfa} kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: 'Created container: sequence-graph-fcbfa' metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05e8787800e58 namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 9999b32d-066a-4193-8524-4c94041b23c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fcbfa} kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Started container sequence-graph-fcbfa metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05e8787f840fc namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: a70c5dda-69ab-484d-9b8b-8860f4e8e501 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fcbfa} kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: Stopping container sequence-graph-fcbfa metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:02Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05efa9585e3a2 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: d1bffb2c-0fb5-4c67-9c20-efe519826ed2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fcbfa} kind: Pod name: sequence-graph-fcbfa-6dfb4df4f4-47wtl namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: a51b549a-c385-40b3-8291-0bba0c89ebfd kind: Event lastTimestamp: "2026-07-08T16:58:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:30Z" name: sequence-graph-fcbfa-6dfb4df4f4-47wtl.18c05efb384085bc namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0413c41f-df44-4063-b1f1-32d97e6a37e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fcbfa-6dfb4df4f4 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 6a52ab1b-84b5-47e9-a940-f563721b97e2 kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: 'Created pod: sequence-graph-fcbfa-6dfb4df4f4-47wtl' metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa-6dfb4df4f4.18c05e8768bf82a1 namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: c5382607-9979-4916-9be3-c1af6320a0a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 5a7a464e-10de-4cb8-96c5-8701b1b80b30 kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Scaled up replica set sequence-graph-fcbfa-6dfb4df4f4 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa.18c05e8767ae7a6d namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: 5b54a5cb-e107-4236-b764-60804e4f143d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: ea1679ef-e1ae-4be9-8342-654d3a6bd55f kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: InferenceGraph [sequence-graph-fcbfa] is Ready metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:48Z" name: sequence-graph-fcbfa.18c05e876d1f43e0 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: 76109e39-6fe3-471b-923c-a72365d51d71 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: d882bf26-0fdf-445a-b447-41de6f49898d kind: Event lastTimestamp: "2026-07-08T16:50:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:03Z" name: sequence-graph-fcbfa.18c05e8ae6a8cf98 namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: c23fa556-e6c0-4778-85ee-7a4b1fd3c33c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: d882bf26-0fdf-445a-b447-41de6f49898d kind: Event lastTimestamp: "2026-07-08T16:50:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:03Z" name: sequence-graph-fcbfa.18c05e8ae70ce258 namespace: kserve-ci-e2e-test resourceVersion: "17452" uid: 15da07bc-1973-4576-b46b-ef28e1440634 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: d882bf26-0fdf-445a-b447-41de6f49898d kind: Event lastTimestamp: "2026-07-08T16:50:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:18Z" name: sequence-graph-fcbfa.18c05e8e65db3b57 namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 39dc4768-1f8d-48dd-b0ed-353d93099d82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: d882bf26-0fdf-445a-b447-41de6f49898d kind: Event lastTimestamp: "2026-07-08T16:50:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:18Z" name: sequence-graph-fcbfa.18c05e8e66437b89 namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: 72423ba6-969f-492c-8f77-8a65f5da7639 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:08:28.334651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-2c597-5f8659dbdc-8ch6c to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:08:28Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8c3b5bffd9 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 6fb34a8b-ceda-43bd-b6fb-202a8c15f0df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-2c597-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8c455ee120 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 49cd8185-3aba-4c04-81d7-3b5630ed475b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:08:29Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:08:29Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8c78ca585f namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: a69f0d28-b655-457a-ade2-b6e67148ba8b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2c597} kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:08:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:29Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8c7a0c8a31 namespace: kserve-ci-e2e-test resourceVersion: "26659" uid: 2f57d65e-d2c1-49a7-931a-baf158044acf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2c597} kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:08:29Z" message: 'Created container: splitter-graph-2c597' metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:29Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8c7d18ed41 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: 112bb856-974a-471a-990c-1b69241204c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2c597} kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:08:29Z" message: Started container splitter-graph-2c597 metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:29Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8c7d8ec168 namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: 963b5aea-376d-4394-9685-0aa3d77f6d93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2c597} kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Stopping container splitter-graph-2c597 metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:38Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8e95158db5 namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 8cfa9a30-48f7-4e42-8941-84ca7b0003fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2c597} kind: Pod name: splitter-graph-2c597-5f8659dbdc-8ch6c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ae370551-afac-4153-9976-5547659ed56f kind: Event lastTimestamp: "2026-07-08T17:09:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:05Z" name: splitter-graph-2c597-5f8659dbdc-8ch6c.18c05f8f298d74e9 namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 19bd3bcb-1e02-4960-b79e-dcbe70aed9ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-2c597-5f8659dbdc namespace: kserve-ci-e2e-test resourceVersion: "26601" uid: 76a11ba2-e2ee-46ea-b435-1e0f9ed97b76 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created pod: splitter-graph-2c597-5f8659dbdc-8ch6c' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: splitter-graph-2c597-5f8659dbdc.18c05f8c3aab37e9 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 1b4f56ae-3e27-4181-ac38-f464969b3922 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-2c597 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 4f494997-4064-4967-8e57-79645345dcf5 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Scaled up replica set splitter-graph-2c597-5f8659dbdc from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: splitter-graph-2c597.18c05f8c39d4f20b namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: 26af6388-53e8-49e5-a99b-8468425676a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-2c597 namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 4a2e6c74-8532-49a1-9b75-1a530599e871 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: InferenceGraph [splitter-graph-2c597] is Ready metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:28Z" name: splitter-graph-2c597.18c05f8c3fe673c2 namespace: kserve-ci-e2e-test resourceVersion: "26619" uid: 77e4c6a4-90c0-422b-81a6-85fc8d12c87f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:09:48.652738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27541" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-ab489-66bcc5c8db-csrnl to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T17:09:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:09:48Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c05f9eeeb0a657 namespace: kserve-ci-e2e-test resourceVersion: "27547" uid: 4bbf2098-c3d0-420d-8105-2cadc4f4a355 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:48Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27544" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:09:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-ab489-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:48Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c05f9ef8d66508 namespace: kserve-ci-e2e-test resourceVersion: "27557" uid: 4135e0e7-e9eb-4ceb-8cc2-f5073c9d51c1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:49Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27550" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:09:49Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:09:49Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c05f9f2c22d104 namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 69b0d008-d3bc-4419-8277-3ec1f2afb17c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab489} kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27544" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:09:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T17:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:49Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c05f9f2d4e5025 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: b7df3be3-9b21-4a0b-b508-5247374f2662 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab489} kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27544" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:09:49Z" message: 'Created container: splitter-graph-ab489' metadata: creationTimestamp: "2026-07-08T17:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:49Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c05f9f2ff46b0a namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: f06b4fd6-6e31-40ed-8503-15679e3b807b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab489} kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27544" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:09:49Z" message: Started container splitter-graph-ab489 metadata: creationTimestamp: "2026-07-08T17:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:49Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c05f9f306a9f1b namespace: kserve-ci-e2e-test resourceVersion: "27572" uid: 60f89911-4895-432e-99ea-5d2f9b51e1f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab489} kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27544" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:18:03Z" message: Stopping container splitter-graph-ab489 metadata: creationTimestamp: "2026-07-08T17:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:03Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c060121881e1b3 namespace: kserve-ci-e2e-test resourceVersion: "31018" uid: 8737451a-1e2a-481d-8f8c-032de3fff290 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:18:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab489} kind: Pod name: splitter-graph-ab489-66bcc5c8db-csrnl namespace: kserve-ci-e2e-test resourceVersion: "27544" uid: 7074dee0-91c1-468c-be9f-f2bfa6ff2c18 kind: Event lastTimestamp: "2026-07-08T17:18:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:18:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:31Z" name: splitter-graph-ab489-66bcc5c8db-csrnl.18c06012c8c74061 namespace: kserve-ci-e2e-test resourceVersion: "31265" uid: 42dbd30d-3966-470e-9e71-5d36db641733 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-ab489-66bcc5c8db namespace: kserve-ci-e2e-test resourceVersion: "27538" uid: 2160b1f0-9320-41aa-ae17-29c678af4543 kind: Event lastTimestamp: "2026-07-08T17:09:48Z" message: 'Created pod: splitter-graph-ab489-66bcc5c8db-csrnl' metadata: creationTimestamp: "2026-07-08T17:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:48Z" name: splitter-graph-ab489-66bcc5c8db.18c05f9eee23d927 namespace: kserve-ci-e2e-test resourceVersion: "27542" uid: d6e8e914-bc5b-4212-a285-87f22f5b543d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-ab489 namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: fb532443-1d34-4e12-b79f-ff2bedd58cdb kind: Event lastTimestamp: "2026-07-08T17:09:48Z" message: Scaled up replica set splitter-graph-ab489-66bcc5c8db from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:48Z" name: splitter-graph-ab489.18c05f9eed7def74 namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: c3fb1caf-acaa-44cc-a348-58cfdc73c3c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-ab489 namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: 7dede621-7914-4c01-86fb-c0152138b136 kind: Event lastTimestamp: "2026-07-08T17:09:48Z" message: InferenceGraph [splitter-graph-ab489] is Ready metadata: creationTimestamp: "2026-07-08T17:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:48Z" name: splitter-graph-ab489.18c05f9ef3441bcf namespace: kserve-ci-e2e-test resourceVersion: "27554" uid: 9a0a28ca-c787-4340-9311-0e87f485bca7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:10:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-ab489 namespace: kserve-ci-e2e-test resourceVersion: "27537" uid: fa7b368c-5bba-4908-b749-670254dfd03d kind: Event lastTimestamp: "2026-07-08T17:10:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:10:33Z" name: splitter-graph-ab489.18c05fa26c1119a8 namespace: kserve-ci-e2e-test resourceVersion: "27899" uid: b109e254-702e-450d-bd33-109a496de5ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:10:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-ab489 namespace: kserve-ci-e2e-test resourceVersion: "27537" uid: fa7b368c-5bba-4908-b749-670254dfd03d kind: Event lastTimestamp: "2026-07-08T17:10:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:10:33Z" name: splitter-graph-ab489.18c05fa26c7c5c31 namespace: kserve-ci-e2e-test resourceVersion: "27900" uid: 0d0313c1-60f0-405d-8045-2e3f13f089c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:57:03.110810Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eecb0d5e3cd namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 95cbd381-67db-4539-8ccb-2a07cf1cd6a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eecca6e10d6 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 596f419c-5ed3-44ab-acdf-91c9c013d0f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eeccbd88845 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: ccc97a83-16ad-4e28-8785-f0721984f13e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eeccece4998 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: bc640d62-a092-4e2e-8115-7904ee78e337 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eeccf407bbf namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 153143d3-326c-4b2f-9d66-2f887d095933 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eeccf4ab31b namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: f4ab606f-58a8-41c1-82c2-ca07653ea929 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eecd30e5083 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 57dbd0af-606d-4f08-82fe-ae1a2d965465 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eecd3813328 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 78883df5-3014-4785-8fb0-bc9409a36d08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:57:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:40Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05eecf4090c6d namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 70e08105-bd8c-47ad-a552-15e5a513be41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05efd0f832ebb namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: 1a41d760-82e8-4931-8518-bcf9389a527e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05efd0f83a892 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 3f80e4e7-65d4-4652-b0e2-febd263ca3b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: f0a878fa-162a-4067-b74e-276407d12efa kind: Event lastTimestamp: "2026-07-08T16:58:15Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:15Z" name: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6.18c05efd7ca1f2d7 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 92569820-5247-43b1-8f95-46717a0e44e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-109dc-predictor-d4554f8b7 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: ad195f57-bee4-4960-b0e2-3c82973dbc71 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Created pod: success-200-isvc-109dc-predictor-d4554f8b7-4wqp6' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor-d4554f8b7.18c05eecb0654955 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 6c724578-1286-449e-bafe-f926ac347b73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: b273b210-2516-4d28-a318-56010f7f32cd kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Scaled up replica set success-200-isvc-109dc-predictor-d4554f8b7 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc-predictor.18c05eecad0ad523 namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: 563f0cc0-053a-409e-9431-72d1b0fc1482 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: b2699d22-e91d-4202-b1aa-d1b606f7a2b4 kind: Event lastTimestamp: "2026-07-08T16:57:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:33Z" name: success-200-isvc-109dc-predictor.18c05ef02bcd93e2 namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: 63819f3a-728b-4d03-81ee-c97ad38be538 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: b2699d22-e91d-4202-b1aa-d1b606f7a2b4 kind: Event lastTimestamp: "2026-07-08T16:57:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:33Z" name: success-200-isvc-109dc-predictor.18c05ef02c12c36d namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 1c7442e2-01b6-42c3-b318-3f5c7a616861 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: b2699d22-e91d-4202-b1aa-d1b606f7a2b4 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: success-200-isvc-109dc-predictor.18c05ef72a84fa22 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 79cb39bc-10e7-4475-b5c2-c8c493e75ec6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-109dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: b2699d22-e91d-4202-b1aa-d1b606f7a2b4 kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:03Z" name: success-200-isvc-109dc-predictor.18c05ef72af34875 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 860444d1-6b3f-429a-920f-cb8cd99c059e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 7694f12f-ed33-45e5-94fc-f3f38be7e1f9 kind: Event lastTimestamp: "2026-07-08T16:57:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:50Z" name: success-200-isvc-109dc.18c05eecad24b18a namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: cc05d7f6-2a20-4bbf-a02e-aec7c96bafb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 7694f12f-ed33-45e5-94fc-f3f38be7e1f9 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-109dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-109dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc.18c05eecadd4fa25 namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 505982dc-49b7-49ce-9730-d47428e48540 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 7694f12f-ed33-45e5-94fc-f3f38be7e1f9 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-109dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-109dc.18c05eecadd557ef namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 666b5624-e566-4e84-9ec8-6bc254b10e58 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-109dc namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 7694f12f-ed33-45e5-94fc-f3f38be7e1f9 kind: Event lastTimestamp: "2026-07-08T16:57:50Z" message: InferenceService [success-200-isvc-109dc] is Ready metadata: creationTimestamp: "2026-07-08T16:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:50Z" name: success-200-isvc-109dc.18c05ef7afc48d14 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: c33013ae-bbc9-4fe3-afe8-d19d627fdc9f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:58:02.961807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:58:02Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efaa03ba70f namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: 7c354f2b-1fbc-41a3-92c6-6cf037bf9e3b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-178bc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:03Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efaaf4adfb7 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 88523b4f-6f3c-46fd-bf5a-b10ae2ca946e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:03Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:58:03Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efadde3e5b9 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: 62624729-d377-4815-809d-90791eb16530 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:04Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efadf3d80c9 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 3ebc1ec2-e849-4ef0-8a08-9aae4b4e7be9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:04Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efae1fea635 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: c07d69f8-238c-4837-8047-5215c4195492 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:04Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efae26b2366 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 114266a2-6844-4564-8f6e-871ad85a0b5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:04Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efae274922e namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 771ca9c0-a619-47b6-af56-abc5a8d06c56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:04Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efae53c1415 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 9c77e520-1417-47d6-af3d-5ca1b81f2c86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:04Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efae5aeb138 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 586d99af-f136-468e-9bcf-f42823739535 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:58:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:40Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05efafe2726e6 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 67551118-4dc8-4a90-b0c6-f032e77107ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05f0afb0e43ac namespace: kserve-ci-e2e-test resourceVersion: "21946" uid: def07d25-803e-47ea-b99a-e18b48e573b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05f0afb0eb42a namespace: kserve-ci-e2e-test resourceVersion: "21948" uid: e6a5bfa5-a610-4a02-a0f2-311f98cfd54e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: eafd9458-b52a-488b-9c7a-804b22f35fed kind: Event lastTimestamp: "2026-07-08T16:59:15Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:15Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt.18c05f0b86829f05 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 7d14385f-968c-4967-9fe7-224790fa5785 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-178bc-predictor-f7f9d7fcf namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: 8221e6f0-18a1-4b42-a799-0d7be3d82f33 kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: 'Created pod: success-200-isvc-178bc-predictor-f7f9d7fcf-nxwzt' metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:02Z" name: success-200-isvc-178bc-predictor-f7f9d7fcf.18c05efa9fc8535c namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: cec0c7ca-e36f-4576-8a19-490c6a467310 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: cfdbee1d-29b4-4a3c-b1c9-209ff4e05d0c kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: Scaled up replica set success-200-isvc-178bc-predictor-f7f9d7fcf from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:02Z" name: success-200-isvc-178bc-predictor.18c05efa9dd20d05 namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: debc8d1a-54b7-45e6-976c-e01701d3e466 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:58:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: d588547d-3fcf-4096-a204-c1e72a7e0850 kind: Event lastTimestamp: "2026-07-08T16:58:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:47Z" name: success-200-isvc-178bc-predictor.18c05efe1c709ed7 namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 8aad4022-c0d4-4bf3-8d04-8de19eb4a46f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:58:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: d588547d-3fcf-4096-a204-c1e72a7e0850 kind: Event lastTimestamp: "2026-07-08T16:58:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:47Z" name: success-200-isvc-178bc-predictor.18c05efe1ce42fb0 namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: ffac53bd-a515-4b9c-90c2-03bd663db116 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: d588547d-3fcf-4096-a204-c1e72a7e0850 kind: Event lastTimestamp: "2026-07-08T16:59:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:02Z" name: success-200-isvc-178bc-predictor.18c05f0899b72216 namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: 93bbd2b6-4315-491c-9192-e4ffd27e87bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-178bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: d588547d-3fcf-4096-a204-c1e72a7e0850 kind: Event lastTimestamp: "2026-07-08T16:59:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:02Z" name: success-200-isvc-178bc-predictor.18c05f089a3755d8 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 85f38c16-23a4-4df2-9219-85779e4daf6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-178bc namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: de612a25-dfd5-4d34-a5c3-7f249f25434f kind: Event lastTimestamp: "2026-07-08T16:58:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:50Z" name: success-200-isvc-178bc.18c05efa9dd1622c namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 330bc841-cf76-41ef-9493-48668d9e8bea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-178bc namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: de612a25-dfd5-4d34-a5c3-7f249f25434f kind: Event lastTimestamp: "2026-07-08T16:58:50Z" message: InferenceService [success-200-isvc-178bc] is Ready metadata: creationTimestamp: "2026-07-08T16:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:50Z" name: success-200-isvc-178bc.18c05f05b9ba6e73 namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: da80f63e-292f-459b-bda3-745c265fe4e3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:07:28.297470Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e40dd3fd0 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 19205c8e-9ac5-49ae-9bd3-9d6eedfe9a63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e5b15b7af namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: 5c5a1146-d511-419e-b163-ec264f6f4bae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e5c634f05 namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: b0944e7b-00c9-4db2-ba20-b63997f51960 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e5f67fd16 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 221751f3-772f-428d-bff2-2cdec9bd7407 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e5fdb1c40 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: 9487706b-869c-4229-b9bd-ff4cf911f2e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e5fee46c2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 5c2586f3-1f0d-476d-9216-bd944d9c6d19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e63d0dc60 namespace: kserve-ci-e2e-test resourceVersion: "26001" uid: dd495d75-85b4-422e-8a64-ff37d6ac3ff5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7e6445ea2d namespace: kserve-ci-e2e-test resourceVersion: "26002" uid: 5b901ea3-2bad-4cf0-aece-8c09fff3f4df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:08:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:06Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f7ea7edccc5 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: dca15351-d678-4ee6-b1df-a20239b95105 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f8e9f2afc94 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 314e10fc-3cee-4c29-af21-5efb2f944525 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f8e9f2b5df0 namespace: kserve-ci-e2e-test resourceVersion: "26817" uid: b2d2c123-b562-4835-8c44-9ee6f3546e53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: c2204dcf-bfa9-4818-8fe4-ba80bb0ab705 kind: Event lastTimestamp: "2026-07-08T17:08:41Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:41Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6.18c05f8f30149e90 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: b78d87b8-667d-42b3-b696-7313d1ca690d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2c597-predictor-6bbc7ccbcc namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: ecb1446f-0cb9-46f9-8efe-d146171dbef0 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: 'Created pod: success-200-isvc-2c597-predictor-6bbc7ccbcc-m66q6' metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor-6bbc7ccbcc.18c05f7e3fbbad50 namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: e41a6e02-0d18-4162-81b4-68c384faf572 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 55d93756-0d49-478a-b055-2421da0f5399 kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Scaled up replica set success-200-isvc-2c597-predictor-6bbc7ccbcc from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-2c597-predictor.18c05f7e3bed07a1 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: b3a38d3c-bc00-4420-a563-1a0ba257023c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: 92959b13-0c42-44b1-a12a-a60952f78f48 kind: Event lastTimestamp: "2026-07-08T17:07:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:58Z" name: success-200-isvc-2c597-predictor.18c05f81ba7b3ed8 namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 1fa42e73-4901-4060-9d56-08e3a7845cc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:07:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: 92959b13-0c42-44b1-a12a-a60952f78f48 kind: Event lastTimestamp: "2026-07-08T17:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:07:58Z" name: success-200-isvc-2c597-predictor.18c05f81bac6bb1f namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1cb59c57-0832-4120-b531-99ba3675fdd7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 92959b13-0c42-44b1-a12a-a60952f78f48 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-2c597-predictor.18c05f88b8d5df91 namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: f96ba0a6-f9d6-4417-8448-7d0bc1e0a69a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2c597-predictor namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 92959b13-0c42-44b1-a12a-a60952f78f48 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-2c597-predictor.18c05f88b93ea302 namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: 04bd11eb-47e5-4112-9408-9d8241565555 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2c597 namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: 95735bbd-b367-4f1e-bb3f-fd290f301a08 kind: Event lastTimestamp: "2026-07-08T17:08:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:16Z" name: success-200-isvc-2c597.18c05f7e3bf0ac7f namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: ffd9f457-bdc3-47ad-bfa0-383ab6c97e39 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2c597 namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 95735bbd-b367-4f1e-bb3f-fd290f301a08 kind: Event lastTimestamp: "2026-07-08T17:08:16Z" message: InferenceService [success-200-isvc-2c597] is Ready metadata: creationTimestamp: "2026-07-08T17:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:16Z" name: success-200-isvc-2c597.18c05f8962c26fe1 namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: c7585741-3a35-4d82-ba96-34496c76f077 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:47:48.170864Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6b7bd65012 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: e0c28c57-1091-45d7-9380-84e01de5ef65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-429af-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6b857d0d79 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 9c87035f-ec6d-449f-b153-81343db47dfc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bb9114075 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 187fa7f6-7fe0-4ce4-a346-0f9d8cae7740 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bba51a658 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: bcc08016-721c-4e50-b497-8fdd765a0352 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bbd1eec7f namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: ca305464-bb7d-4575-82ec-01225f102027 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bbd7d27cf namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 569e78fc-47c4-4518-8a04-3d08f1be284a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bbd85df6b namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: dca71c5f-51bc-4cf5-82e6-d7245f7b7bf0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bc119376c namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 9b89f984-fe78-4ed6-9d35-df6f6d35b4bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:47:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:49Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6bc17c11c5 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: 1a8c6329-0bab-4749-b5d1-bea4661bd430 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:48:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:26Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05e6c1ce68a4e namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: e5f3ad0f-76aa-4491-96af-5a622b4b54a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05eecadeede98 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: ae20ffd1-afe3-4fc7-88e5-a3916517b0f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 8ddf3e70-7c90-42fa-9d62-2037f9ec0244 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:03Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk.18c05eecadef3955 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: b1d571c3-5f0c-44f7-a975-98e7d902bb9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-429af-predictor-7c6dbbb6f7 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: fa7a6255-ca6b-40d6-9025-910c785f0950 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Created pod: success-200-isvc-429af-predictor-7c6dbbb6f7-pjjwk' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-429af-predictor-7c6dbbb6f7.18c05e6b7b5ea09f namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: a7ab843b-6f40-46be-8546-888ce79fa530 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 39d8a95f-4f61-4b86-bb52-2b1d9755cd0c kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Scaled up replica set success-200-isvc-429af-predictor-7c6dbbb6f7 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-429af-predictor.18c05e6b79c5d4c6 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 992046e0-7b28-43ab-bed1-3e9d42b5a7e4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:48:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: fbefe0d6-bb51-44c1-bb22-5c3fcc3b6e64 kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:33Z" name: success-200-isvc-429af-predictor.18c05e6ef85f3d0f namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 7e1ce0c2-261f-40d6-b52c-553448988572 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:48:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: fbefe0d6-bb51-44c1-bb22-5c3fcc3b6e64 kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:33Z" name: success-200-isvc-429af-predictor.18c05e6ef8ccb17a namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 45cfb770-9846-4e87-ad33-4752f9aa94b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: fbefe0d6-bb51-44c1-bb22-5c3fcc3b6e64 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-429af-predictor.18c05e797509e1d1 namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: b6fbe1cf-20bf-415d-a753-4604973c7879 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-429af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: fbefe0d6-bb51-44c1-bb22-5c3fcc3b6e64 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-429af-predictor.18c05e79756f54fe namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: 6262a3b1-bddf-4ef3-a046-8ea721f86e66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 1267d9b3-8ccc-4fd8-afc6-8d10952f8e16 kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:37Z" name: success-200-isvc-429af.18c05e6b79cb6efc namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 987d1a98-f01f-458d-8f79-8e0146550c83 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 1267d9b3-8ccc-4fd8-afc6-8d10952f8e16 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Failed to update status for InferenceService "success-200-isvc-429af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-429af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-429af.18c05e6b7a490b70 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 7bc88034-4c8d-4648-adbc-b284b66fa23c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 1267d9b3-8ccc-4fd8-afc6-8d10952f8e16 kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-429af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-429af.18c05e6b7a495a33 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: eab49c98-35f7-4f20-8b0b-fa10a07e528f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-429af namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 1267d9b3-8ccc-4fd8-afc6-8d10952f8e16 kind: Event lastTimestamp: "2026-07-08T16:48:36Z" message: InferenceService [success-200-isvc-429af] is Ready metadata: creationTimestamp: "2026-07-08T16:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:36Z" name: success-200-isvc-429af.18c05e76d8944476 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: 60dc9c95-f993-4d0c-96f8-6ff20696e4f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:58:13.445419Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-806a8-predictor-d7dcd8679-j829p to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd111af0f3 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 46ddcec4-b775-4f3f-91e0-a3dcd77628e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd2aba740f namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 4845d9f1-f140-4bb5-a89f-33584db5ac57 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd2c0e4e05 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: 8c3e12b6-1d61-420a-b9e3-0a6104c8b2de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd2ec0ec2a namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: a62279d2-40fe-4aeb-980d-90257e46da0d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd2f320499 namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 7b12d937-9e79-46ee-a194-ed7286a24893 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd2f3a85fc namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: 9252695c-603f-4e03-b4e9-65d0c997f397 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd32631cf0 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 54f81f71-495e-4803-b9bd-25f7eeb3cfd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T16:58:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:14Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd32d20d32 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: d799646e-225e-477f-b727-451556cddd83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T17:07:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:30Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05efd5618d993 namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: 7e3b7573-605b-4e1d-a452-c36bfb4a90a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05f7e3b86f886 namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: 845ede77-756e-43a4-851f-e5d3c4a27f92 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T17:07:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:28Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05f7e3b879234 namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: c2140542-117a-4cd0-bba9-e7d1ca18afe3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-806a8-predictor-d7dcd8679-j829p namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 16b45dea-ae5b-4b61-9b05-209978bc4a1f kind: Event lastTimestamp: "2026-07-08T17:07:30Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:07:30Z" name: success-200-isvc-806a8-predictor-d7dcd8679-j829p.18c05f7ecaa69126 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: c5647c77-081e-48aa-b376-53ea5f0d59f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-806a8-predictor-d7dcd8679 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 9f6aff90-2863-4bb9-a228-63b71f3f8d51 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'Created pod: success-200-isvc-806a8-predictor-d7dcd8679-j829p' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor-d7dcd8679.18c05efd10af6c19 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 45eadb42-0478-4331-bd3c-22e0ebf66da6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 26fbf48d-1f00-4743-86ac-dd6fb23c3ce0 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: Scaled up replica set success-200-isvc-806a8-predictor-d7dcd8679 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8-predictor.18c05efd0ed397d1 namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: f8cde1e5-97a6-4e8c-87fe-a0e37ad1babe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 48c5fc30-ba0e-4d86-bb17-9ef923c181a6 kind: Event lastTimestamp: "2026-07-08T16:58:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:43Z" name: success-200-isvc-806a8-predictor.18c05f008e16f273 namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: 072cac03-2411-4616-aa61-44638c628aa2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 48c5fc30-ba0e-4d86-bb17-9ef923c181a6 kind: Event lastTimestamp: "2026-07-08T16:58:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:43Z" name: success-200-isvc-806a8-predictor.18c05f008eccc28d namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: 433bcaf9-b20d-4691-b304-47c10fb23a40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 48c5fc30-ba0e-4d86-bb17-9ef923c181a6 kind: Event lastTimestamp: "2026-07-08T16:59:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:28Z" name: success-200-isvc-806a8-predictor.18c05f078cfbf0ce namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 34974560-6883-4eba-bde5-f57fcc9530ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-806a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 48c5fc30-ba0e-4d86-bb17-9ef923c181a6 kind: Event lastTimestamp: "2026-07-08T16:59:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:28Z" name: success-200-isvc-806a8-predictor.18c05f078d6af837 namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: fc9509b3-5262-489f-88d5-893fbfd90082 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 421eaed4-9bea-4569-8741-2e05f3a72633 kind: Event lastTimestamp: "2026-07-08T16:59:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:00Z" name: success-200-isvc-806a8.18c05efd0e5d88ab namespace: kserve-ci-e2e-test resourceVersion: "21748" uid: 5d1ea2d5-e988-418c-8c01-3cc9f39ee736 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 421eaed4-9bea-4569-8741-2e05f3a72633 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-806a8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-806a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8.18c05efd0eb75720 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: d1034477-2665-46b6-a8d5-0dcccdbee817 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 421eaed4-9bea-4569-8741-2e05f3a72633 kind: Event lastTimestamp: "2026-07-08T16:58:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-806a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:13Z" name: success-200-isvc-806a8.18c05efd0eb7a711 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 67f49f8f-3932-4e85-91c4-d6f966d1648d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-806a8 namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 421eaed4-9bea-4569-8741-2e05f3a72633 kind: Event lastTimestamp: "2026-07-08T16:59:00Z" message: InferenceService [success-200-isvc-806a8] is Ready metadata: creationTimestamp: "2026-07-08T16:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:00Z" name: success-200-isvc-806a8.18c05f0812a53880 namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 746f0616-1a0c-4152-86f6-2de0327cb64d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:08:38.608924Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ab489-predictor-5448cc6d6f-rl89h to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8e9fc0ee96 namespace: kserve-ci-e2e-test resourceVersion: "26819" uid: 1596f4ba-9608-413a-ad4b-f0c648af3c64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ab489-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8eaf634fba namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d3a8b3fe-b946-416f-b6df-1e6a38f12fed reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8edd40ba26 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 3fe5c459-89b2-4417-a82a-ea6238c3f1f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8ede9b8556 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: cb5e5280-d2ef-4a7f-b4db-f956a7554387 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8ee1c966d3 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: df904a33-151a-475d-95b3-8140dfc297b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8ee2351411 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: b0376b58-d8a4-418d-90f7-f638470cabb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8ee23eb02c namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: e13f6a49-352a-430d-b076-833b70b38c11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8ee61a4d13 namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: 2e2714b9-6df4-45a2-a842-3fd5d8e6a8b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:08:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:39Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8ee68d1a8d namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: 54c13213-54e9-4a59-906f-82f58ce449c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T17:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:09:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:26Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c05f8f082c633c namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: 0be80c5d-d859-4228-a1a2-709a26483545 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:18:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:03Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c060121feecdab namespace: kserve-ci-e2e-test resourceVersion: "31035" uid: c90dba40-257d-4d45-8133-22294991ab41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 57dc77a7-b03c-4dad-a43b-96d7981d9400 kind: Event lastTimestamp: "2026-07-08T17:18:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:18:03Z" name: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h.18c060121fef3da1 namespace: kserve-ci-e2e-test resourceVersion: "31036" uid: 5ffabf9a-c812-4ba3-83eb-39275b119bfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ab489-predictor-5448cc6d6f namespace: kserve-ci-e2e-test resourceVersion: "26797" uid: 7e491649-d040-4d3a-a2e5-982e5c9eeb75 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'Created pod: success-200-isvc-ab489-predictor-5448cc6d6f-rl89h' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-ab489-predictor-5448cc6d6f.18c05f8e9f5b4b4b namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: dcf107e2-03aa-4b36-aab6-cc581e634c4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 3858f79a-75fa-4365-8f42-ec38b8003fb4 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: Scaled up replica set success-200-isvc-ab489-predictor-5448cc6d6f from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-ab489-predictor.18c05f8e9d6a6530 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 3be0fa1b-f01c-4a70-aefc-5e7280bddfa9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 2ceed403-1611-4f3e-b275-355edb43371f kind: Event lastTimestamp: "2026-07-08T17:08:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:53Z" name: success-200-isvc-ab489-predictor.18c05f921c1cad49 namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: 4ee05360-396a-4722-8024-df01bc5a12e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 2ceed403-1611-4f3e-b275-355edb43371f kind: Event lastTimestamp: "2026-07-08T17:08:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:53Z" name: success-200-isvc-ab489-predictor.18c05f921c74a84c namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: 9fc31936-2098-4abb-a93c-52236cfc4a30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:09:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 2ceed403-1611-4f3e-b275-355edb43371f kind: Event lastTimestamp: "2026-07-08T17:10:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:10:08Z" name: success-200-isvc-ab489-predictor.18c05f959b3e1c70 namespace: kserve-ci-e2e-test resourceVersion: "27721" uid: ad56fc94-4544-4103-aa45-f330d8de5bd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:09:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab489-predictor namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 2ceed403-1611-4f3e-b275-355edb43371f kind: Event lastTimestamp: "2026-07-08T17:10:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:10:08Z" name: success-200-isvc-ab489-predictor.18c05f959ba2c4b6 namespace: kserve-ci-e2e-test resourceVersion: "27722" uid: 96c0cc1f-daef-43e4-948d-d6786db46dcd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 4f37249e-0781-4187-9ffd-bb3a35769cf5 kind: Event lastTimestamp: "2026-07-08T17:09:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:36Z" name: success-200-isvc-ab489.18c05f8e9db05d0b namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 19f27505-3098-4e28-8785-36447131d7b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 4f37249e-0781-4187-9ffd-bb3a35769cf5 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-ab489": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ab489": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-ab489.18c05f8e9e2d6970 namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: 80f44cce-ecd3-484b-99c0-9c925de9a58a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 4f37249e-0781-4187-9ffd-bb3a35769cf5 kind: Event lastTimestamp: "2026-07-08T17:08:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ab489": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:38Z" name: success-200-isvc-ab489.18c05f8e9e2dc894 namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: 7692227d-a653-42bb-a8a3-2f66c3275604 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab489 namespace: kserve-ci-e2e-test resourceVersion: "27436" uid: 4f37249e-0781-4187-9ffd-bb3a35769cf5 kind: Event lastTimestamp: "2026-07-08T17:09:36Z" message: InferenceService [success-200-isvc-ab489] is Ready metadata: creationTimestamp: "2026-07-08T17:09:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:36Z" name: success-200-isvc-ab489.18c05f9c170882a3 namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: 6ecea9de-7302-43e0-8638-5a00708d6883 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:59:13.258437Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0afe3d1cd2 namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: a6277823-3b32-4b4c-ba6f-6fb09ac103a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b2f8c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b0a0c8a6c namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 1d577321-7590-4072-af93-782a61af0e65 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b3b84a1a1 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: 9b9fd0be-3faa-4a76-a225-505efade2ffd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b3cbe1460 namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 416c08fc-ff2e-4a2d-aef7-c8f01a672d85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b3fb1a92d namespace: kserve-ci-e2e-test resourceVersion: "22062" uid: ef884ff9-482e-4aa3-b2aa-1a3a067c0ffb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b403dab85 namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: dbf285a0-e03a-4ad0-b900-e07f0bb2095d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b404683d6 namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 7e62423b-84f0-4a4e-b26d-2e02f6be3763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b430ae5aa namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 9f7f4540-67d7-4fa5-8541-ab5ea1210cec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:14Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b4378111f namespace: kserve-ci-e2e-test resourceVersion: "22066" uid: f110ac71-843f-4ba5-9cc7-e85a00db3ed5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T16:59:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:51Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f0b9be00cfc namespace: kserve-ci-e2e-test resourceVersion: "22384" uid: de833303-ea9d-4be7-a00d-1262ca89d455 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T17:08:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:27Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f8c2483a1b8 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: e1b7e2f5-7fb9-4e1b-ae1f-0672c8556f9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: e717f006-ba23-401e-9efe-422da3ea256e kind: Event lastTimestamp: "2026-07-08T17:08:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:27Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c.18c05f8c2484e1ea namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 394898d2-3a74-4771-b626-762c593ba54b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2f8c-predictor-867ddf48ff namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: ed2ac2cc-1e36-4526-8108-d6a7f12fd5bd kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Created pod: success-200-isvc-b2f8c-predictor-867ddf48ff-z8f8c' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-b2f8c-predictor-867ddf48ff.18c05f0afdcaea9a namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: 67dbbec4-4ab4-4f1c-aac9-e413a1c97f98 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: 1191c38e-165d-4314-92d2-b53923807f66 kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: Scaled up replica set success-200-isvc-b2f8c-predictor-867ddf48ff from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-b2f8c-predictor.18c05f0afbfc6fa6 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: 42f614b7-9e24-481d-87db-243e4694cfca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21947" uid: b5acb73c-b000-4378-9c47-1924a2505fe0 kind: Event lastTimestamp: "2026-07-08T16:59:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:43Z" name: success-200-isvc-b2f8c-predictor.18c05f0e7b0ace54 namespace: kserve-ci-e2e-test resourceVersion: "22316" uid: c7a82604-c0ef-4797-91b4-82787efb38a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21947" uid: b5acb73c-b000-4378-9c47-1924a2505fe0 kind: Event lastTimestamp: "2026-07-08T16:59:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:43Z" name: success-200-isvc-b2f8c-predictor.18c05f0e7b7c1b5b namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: a1a2904e-4739-4aee-b9ef-59972e4eb0dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:59:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: b5acb73c-b000-4378-9c47-1924a2505fe0 kind: Event lastTimestamp: "2026-07-08T17:00:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:28Z" name: success-200-isvc-b2f8c-predictor.18c05f1579cba56a namespace: kserve-ci-e2e-test resourceVersion: "22709" uid: 29e1e7e5-55b1-48fd-9b0b-5da5b44a155a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:59:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2f8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: b5acb73c-b000-4378-9c47-1924a2505fe0 kind: Event lastTimestamp: "2026-07-08T17:00:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:28Z" name: success-200-isvc-b2f8c-predictor.18c05f157a388a34 namespace: kserve-ci-e2e-test resourceVersion: "22710" uid: 2e947bbb-7ef2-4de4-a38c-f671ff58414b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 7bd9bb21-75a6-4aea-8ee9-03b14c7452cd kind: Event lastTimestamp: "2026-07-08T17:00:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:00:02Z" name: success-200-isvc-b2f8c.18c05f0afbf39fc1 namespace: kserve-ci-e2e-test resourceVersion: "22479" uid: cbeaee68-4e95-4d81-b3e7-3000eb2798aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 7bd9bb21-75a6-4aea-8ee9-03b14c7452cd kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-b2f8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2f8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-b2f8c.18c05f0afc41df4c namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 42e83d2e-ea60-4ddb-ace6-a4ebd3609a5b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 7bd9bb21-75a6-4aea-8ee9-03b14c7452cd kind: Event lastTimestamp: "2026-07-08T16:59:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2f8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:13Z" name: success-200-isvc-b2f8c.18c05f0afc423de1 namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: fcb051b2-76c2-44f7-abcb-7e3b8789b66f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2f8c namespace: kserve-ci-e2e-test resourceVersion: "22476" uid: 7bd9bb21-75a6-4aea-8ee9-03b14c7452cd kind: Event lastTimestamp: "2026-07-08T17:00:02Z" message: InferenceService [success-200-isvc-b2f8c] is Ready metadata: creationTimestamp: "2026-07-08T17:00:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:00:02Z" name: success-200-isvc-b2f8c.18c05f165a1dc4ca namespace: kserve-ci-e2e-test resourceVersion: "22477" uid: ee6a9a89-fe84-46be-9401-67fb58293efd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:08:28.030021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d09b9-predictor-75cd6867df-q6mgb to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c2933bed1 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: bf6de1c6-e1e9-4021-9af6-d4461ccbd4df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c4311c5bf namespace: kserve-ci-e2e-test resourceVersion: "26621" uid: c64b97bc-b448-456a-b601-0e5a8c5bc341 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c452a58d2 namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: 2f43925e-1d12-4a2e-b7c8-33731738d642 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c4820d366 namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: fff3d26e-4829-4945-960e-57b0ce7efc3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c489d9afc namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 50e3bfe0-673d-4804-afd6-40ff0b7afdbc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c48a95c44 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: a33d7880-3754-4575-b624-9df2be521ab0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c4b4d8906 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: f138fb86-8674-48eb-aa58-f52dceee1e40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c4bc8c2bd namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 49d79420-5793-476f-958b-c90154143a42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:09:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:05Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c05f8c74bc7c20 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: 2000e1bc-c83d-46ee-9a06-d6eb46998587 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:25:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:25:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d09b9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:51Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c0607e39e53661 namespace: kserve-ci-e2e-test resourceVersion: "34329" uid: 76e5622b-2c7a-4912-be15-151ce745f9b7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:25:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:48Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c0607e6d4c0f94 namespace: kserve-ci-e2e-test resourceVersion: "34267" uid: 3d9175ea-2127-4a2d-9551-ea6d4c62206d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:25:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:48Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c0607e6d4c73a8 namespace: kserve-ci-e2e-test resourceVersion: "34268" uid: 38305937-44c8-4a66-89db-b822f5486be8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7c6494-5c28-4353-8f42-620c8e8d277b kind: Event lastTimestamp: "2026-07-08T17:25:50Z" message: 'Readiness probe failed: Get "https://10.133.0.45:8643/healthz": dial tcp 10.133.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:50Z" name: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb.18c0607ed58343d1 namespace: kserve-ci-e2e-test resourceVersion: "34318" uid: 7f530008-b30a-40a9-8d51-1fe984bb9132 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d09b9-predictor-75cd6867df namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 3a398c83-11f2-40bf-933c-a9cb447988e0 kind: Event lastTimestamp: "2026-07-08T17:08:28Z" message: 'Created pod: success-200-isvc-d09b9-predictor-75cd6867df-q6mgb' metadata: creationTimestamp: "2026-07-08T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:28Z" name: success-200-isvc-d09b9-predictor-75cd6867df.18c05f8c285435e1 namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: 33026111-929e-494e-8d2f-758614acc6b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: 987beca7-5b6f-4eee-bbca-7a2c39e3d8be kind: Event lastTimestamp: "2026-07-08T17:08:27Z" message: Scaled up replica set success-200-isvc-d09b9-predictor-75cd6867df from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:27Z" name: success-200-isvc-d09b9-predictor.18c05f8c26867520 namespace: kserve-ci-e2e-test resourceVersion: "26529" uid: 2b11fa74-a087-47b2-8ea5-cc893aa19dd3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c05a931d-c1e3-4233-8322-8f9155c1f308 kind: Event lastTimestamp: "2026-07-08T17:08:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:58Z" name: success-200-isvc-d09b9-predictor.18c05f8fa4bb028b namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: c73d6858-36e3-4c4f-b232-b91e3d2562b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:08:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c05a931d-c1e3-4233-8322-8f9155c1f308 kind: Event lastTimestamp: "2026-07-08T17:08:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:08:58Z" name: success-200-isvc-d09b9-predictor.18c05f8fa520a58b namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: 92f3c0bb-aa5f-48a2-8a86-09b649f2a821 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:09:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26931" uid: c05a931d-c1e3-4233-8322-8f9155c1f308 kind: Event lastTimestamp: "2026-07-08T17:09:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:43Z" name: success-200-isvc-d09b9-predictor.18c05f96a3bec1ff namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: ff731b95-f63e-4d89-a7d5-12d75bf1f4b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:09:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d09b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26931" uid: c05a931d-c1e3-4233-8322-8f9155c1f308 kind: Event lastTimestamp: "2026-07-08T17:09:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:43Z" name: success-200-isvc-d09b9-predictor.18c05f96a4a7b648 namespace: kserve-ci-e2e-test resourceVersion: "27488" uid: 2ed473d0-05b9-405c-b339-dbfed424f8f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 145fe0dd-493e-4afd-b9d4-6077a0248d9a kind: Event lastTimestamp: "2026-07-08T17:09:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:15Z" name: success-200-isvc-d09b9.18c05f8c2654b1e6 namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: 40b95bb3-87a2-43f4-94a9-9c3fbeb8c7f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 145fe0dd-493e-4afd-b9d4-6077a0248d9a kind: Event lastTimestamp: "2026-07-08T17:08:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-d09b9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d09b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:27Z" name: success-200-isvc-d09b9.18c05f8c26d83148 namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: a714bc5e-7b2d-455a-b302-302f23ad0a8c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:08:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 145fe0dd-493e-4afd-b9d4-6077a0248d9a kind: Event lastTimestamp: "2026-07-08T17:08:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d09b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:08:27Z" name: success-200-isvc-d09b9.18c05f8c26d89970 namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: 3f12b85b-d773-4a97-8f4b-d5b12ddb8591 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27215" uid: 145fe0dd-493e-4afd-b9d4-6077a0248d9a kind: Event lastTimestamp: "2026-07-08T17:09:15Z" message: InferenceService [success-200-isvc-d09b9] is Ready metadata: creationTimestamp: "2026-07-08T17:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:15Z" name: success-200-isvc-d09b9.18c05f973111e4c6 namespace: kserve-ci-e2e-test resourceVersion: "27216" uid: a2b30207-2754-419f-8727-c6a0dda3b74f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:18.099577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:18Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e56832b8b21 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 20c12670-c6f6-4c8c-a573-5fa64e680bed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:18Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e569cea89b7 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 258e158c-564a-4030-8591-b00554930a2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:18Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e569e51674e namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: c717ba0b-027c-4cba-8bd1-057de2ff3c0b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" in 15.921s (15.921s including waiting). Image size: 1335681123 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5a534c8ff5 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 38730945-3ef7-4c92-91c1-9eb1caad7f62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5a56cd3447 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 1b6808b1-a256-4f34-9adb-fdae93024f17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5a5741092c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: e642aaea-e2ae-49d7-82fa-a2108fc7b9e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:34Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5a574cac3f namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 3e28224a-78b7-463f-8e10-b08cecbb4e99 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.533s (2.533s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5aee4b25c6 namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 58588321-87ed-4e82-af22-e071e86618bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5af1923340 namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: adfc144c-9869-428b-9305-350636797201 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5af22aca08 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 867c2bf6-0b01-4aeb-b510-304982e62e31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:47:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:13Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e5b0c362cc8 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 3931c389-b7f4-4d3d-9221-25d0dc829a2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e6b76ed0a35 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: b237468e-7386-4ce1-b449-4f2baa28efdb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e6b76edbe06 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: a957906e-48d0-48db-a07f-87cf237f5121 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: e0161d25-bd08-40da-84ff-a62b4758a1ad kind: Event lastTimestamp: "2026-07-08T16:47:48Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:48Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t.18c05e6b94912745 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 157f939c-5861-45c6-9032-3de95d5cea0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f6c4d-predictor-58f8f4fd9f namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: 75112a4a-99f2-469b-bca8-d9135bd24235 kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: 'Created pod: success-200-isvc-f6c4d-predictor-58f8f4fd9f-bbf8t' metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: success-200-isvc-f6c4d-predictor-58f8f4fd9f.18c05e56829dc7a7 namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: fe2749ac-aadc-4086-bfd2-77dc3a6ef746 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 84fe8472-7cfe-4681-9583-6032bdacfbff kind: Event lastTimestamp: "2026-07-08T16:46:18Z" message: Scaled up replica set success-200-isvc-f6c4d-predictor-58f8f4fd9f from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:18Z" name: success-200-isvc-f6c4d-predictor.18c05e567db5f324 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 8e47f0d8-674e-451f-b199-2fb0bb6d5133 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 7a876dd7-4ebd-4a8a-b6c3-171ec323d860 kind: Event lastTimestamp: "2026-07-08T16:47:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:03Z" name: success-200-isvc-f6c4d-predictor.18c05e59fc6f9403 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 91c62a5b-e060-4f38-acca-52096999ba5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 7a876dd7-4ebd-4a8a-b6c3-171ec323d860 kind: Event lastTimestamp: "2026-07-08T16:47:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:03Z" name: success-200-isvc-f6c4d-predictor.18c05e59fcd68c62 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: a07ca64c-adbe-4943-ae05-d472b08dd5fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 7a876dd7-4ebd-4a8a-b6c3-171ec323d860 kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: success-200-isvc-f6c4d-predictor.18c05e647919ce3a namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 0de353c0-d343-471f-b7c3-c7a2fbff6d12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 7a876dd7-4ebd-4a8a-b6c3-171ec323d860 kind: Event lastTimestamp: "2026-07-08T16:47:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:33Z" name: success-200-isvc-f6c4d-predictor.18c05e64798784f1 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 69d25bf2-4ef5-4f1f-89cb-7d51bd46070b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:46:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6c4d namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 971398f6-f66b-430d-82b9-742650cef09d kind: Event lastTimestamp: "2026-07-08T16:47:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:23Z" name: success-200-isvc-f6c4d.18c05e567e188a4e namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 4a9bb251-7d7a-4c54-87ba-01edb724ca0c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6c4d namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 971398f6-f66b-430d-82b9-742650cef09d kind: Event lastTimestamp: "2026-07-08T16:47:23Z" message: InferenceService [success-200-isvc-f6c4d] is Ready metadata: creationTimestamp: "2026-07-08T16:47:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:23Z" name: success-200-isvc-f6c4d.18c05e65c8a1fddc namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: 707cb9f6-db75-46ae-850f-da87fba61ac6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:48:48.086550Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fcbfa-predictor-855b5b6765-49fnz to ip-10-0-135-143.ec2.internal metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e796f171aa5 namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: a29f5da6-8705-425a-81c6-f7def48c10df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e798842eb1f namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: a44e1f00-7349-47fe-9cbb-40c50a1c1e57 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e798981b458 namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: adfbe8d4-ec03-4c2e-aefc-ca7545a35a90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e798c38408a namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: a05d0301-9760-4889-8267-84464a9bd091 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e798ca4dd8b namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: bd5f4028-b7dc-4bbf-94b7-fe9e0894d824 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e798cb09034 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 73d92277-f1c7-4200-866b-8f6f418542b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e79909ee394 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: f2a867e5-04a2-47e7-9098-e551560f1fe3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e79910615e3 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: ba521d25-935e-4b0f-b4d3-071c3dd17d0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:58:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:06Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05e79e8decb80 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 437d141c-fd63-47e9-87d9-cdeaade26501 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:02Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05efa9c20b8c0 namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: 217e5647-ea8d-4fec-8592-4eec77e180b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:58:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:02Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05efa9c2131c8 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: a0ec6464-1ac6-41ed-bc33-254a5d40e4a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 6641dc6f-d7e0-4464-8f9d-a04e461ea2ed kind: Event lastTimestamp: "2026-07-08T16:58:06Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:06Z" name: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz.18c05efb5d4b2f93 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 5aa96fe4-7d5e-4a6f-8898-8b354f121a55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-143.ec2.internal source: component: kubelet host: ip-10-0-135-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fcbfa-predictor-855b5b6765 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: e29a7cae-225d-4502-90de-7a0b7a218956 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created pod: success-200-isvc-fcbfa-predictor-855b5b6765-49fnz' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor-855b5b6765.18c05e796e981463 namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 9fd720c4-0e95-4e59-9ec0-eea855dd2bca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: f5a0d30c-3a09-4222-82d3-5ea9f5f5815c kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Scaled up replica set success-200-isvc-fcbfa-predictor-855b5b6765 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: success-200-isvc-fcbfa-predictor.18c05e796c76f1b9 namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: dd0a3c77-60ec-4598-92ca-dde5491a11c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 215093e3-ccb6-49f0-9115-05ac786298d8 kind: Event lastTimestamp: "2026-07-08T16:49:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:18Z" name: success-200-isvc-fcbfa-predictor.18c05e7ceb77c8d3 namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: 61f79770-5f2a-4a27-ba0b-6811d5e48d34 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 215093e3-ccb6-49f0-9115-05ac786298d8 kind: Event lastTimestamp: "2026-07-08T16:49:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:18Z" name: success-200-isvc-fcbfa-predictor.18c05e7cebdf50e8 namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: a48c424f-b4e5-405b-8fbf-383033cce648 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:49:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 215093e3-ccb6-49f0-9115-05ac786298d8 kind: Event lastTimestamp: "2026-07-08T16:50:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:03Z" name: success-200-isvc-fcbfa-predictor.18c05e83e9dd1604 namespace: kserve-ci-e2e-test resourceVersion: "17453" uid: d605605e-cdb3-4583-9f97-1b0626df3210 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:49:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fcbfa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 215093e3-ccb6-49f0-9115-05ac786298d8 kind: Event lastTimestamp: "2026-07-08T16:50:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:03Z" name: success-200-isvc-fcbfa-predictor.18c05e83ea2b23aa namespace: kserve-ci-e2e-test resourceVersion: "17454" uid: 0a269cc2-dbc5-46e7-bffe-66eb874ec541 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 63fbf011-63c5-4bd7-80ee-0dafdfe88eca kind: Event lastTimestamp: "2026-07-08T16:49:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:36Z" name: success-200-isvc-fcbfa.18c05e796c78557a namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: 285e8c1d-2e39-43e6-a3f0-c8ec1dfb4118 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fcbfa namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 63fbf011-63c5-4bd7-80ee-0dafdfe88eca kind: Event lastTimestamp: "2026-07-08T16:49:36Z" message: InferenceService [success-200-isvc-fcbfa] is Ready metadata: creationTimestamp: "2026-07-08T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:36Z" name: success-200-isvc-fcbfa.18c05e84a3d94cde namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: ae0d7570-dbdc-4505-8768-aa5425ef8dae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:48:48.220327Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-429af-6d6c9b7b59-jzl22 to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05e797710615e namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: de062b9c-cd77-48f4-a530-dad1664b4e6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05e7991e31456 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: d152997b-1833-4717-bea3-32b8ecc17f14 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-429af} kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05e7993060876 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 8cafe75c-2bb8-4228-bbc8-9553fa2c5df9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-429af} kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created container: switch-graph-429af' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05e7995a9c37e namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: e7d2d400-da7e-4db6-ac56-3a9152a3d55b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-429af} kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Started container switch-graph-429af metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05e79961e81c2 namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: db617d70-6785-48c7-a8cc-90ff44a33482 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-429af} kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: "2026-07-08T16:57:02Z" message: Stopping container switch-graph-429af metadata: creationTimestamp: "2026-07-08T16:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:02Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05eeca5620257 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: ed0db63c-3457-4924-b921-e9b8cdf71854 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-429af} kind: Pod name: switch-graph-429af-6d6c9b7b59-jzl22 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 8ce16306-c24a-4aba-a662-fa67268316b4 kind: Event lastTimestamp: "2026-07-08T16:57:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:30Z" name: switch-graph-429af-6d6c9b7b59-jzl22.18c05eed3207ce7f namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: b96c72ce-4c4c-4126-9cfb-ed01b062614e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-429af-6d6c9b7b59 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: d62f2d37-c7fa-4088-a575-6f8d40a7580a kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: 'Created pod: switch-graph-429af-6d6c9b7b59-jzl22' metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af-6d6c9b7b59.18c05e79767cef01 namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: e570b8b8-9a5f-479d-b1c5-7c661ffebc7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-429af namespace: kserve-ci-e2e-test resourceVersion: "16735" uid: f223d266-d811-43e7-b491-f6b31dbbc789 kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: Scaled up replica set switch-graph-429af-6d6c9b7b59 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af.18c05e797599613d namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 6ca2f894-32a2-4231-8e00-259f528c4466 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-429af namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 3a24a688-2097-42d3-849e-4e138fe42cbc kind: Event lastTimestamp: "2026-07-08T16:48:48Z" message: InferenceGraph [switch-graph-429af] is Ready metadata: creationTimestamp: "2026-07-08T16:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:48Z" name: switch-graph-429af.18c05e797d198feb namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: ffcd34f5-05d9-4ca4-8bf4-317ae4e99dc0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-429af namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: fd9e54fd-3622-4cff-8954-6b4b0f737fcf kind: Event lastTimestamp: "2026-07-08T16:49:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:03Z" name: switch-graph-429af.18c05e7cf44222af namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: c4e6aa0e-8b51-4aa8-8a54-8a25f1cbff95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-429af namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: fd9e54fd-3622-4cff-8954-6b4b0f737fcf kind: Event lastTimestamp: "2026-07-08T16:49:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:03Z" name: switch-graph-429af.18c05e7cf4b0c9e9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: b2c5c275-0ddf-4706-9957-4fe6d536ff31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-429af namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: fd9e54fd-3622-4cff-8954-6b4b0f737fcf kind: Event lastTimestamp: "2026-07-08T16:49:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:18Z" name: switch-graph-429af.18c05e80736ea050 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 88bfc085-e846-4ac2-83ff-cbdc5a18e84b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-429af namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: fd9e54fd-3622-4cff-8954-6b4b0f737fcf kind: Event lastTimestamp: "2026-07-08T16:49:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:18Z" name: switch-graph-429af.18c05e8073bf0673 namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: 724ca611-5779-4420-8d88-be16def4c052 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:09:28.094297Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27331" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d09b9-bd8cfc589-h94lt to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T17:09:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:09:28Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c05f9a254fc315 namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: 3ac15bec-c3ba-4c01-814b-a5e98f2bd694 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:28Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:09:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d09b9-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:28Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c05f9a2f50aaea namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: 7c74ef67-6730-4d38-882d-e186dc0dea4e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:09:29Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:09:29Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c05f9a63532c67 namespace: kserve-ci-e2e-test resourceVersion: "27359" uid: 5d5de753-3e92-4a95-94d7-809ce423ac86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d09b9} kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:09:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T17:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:29Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c05f9a64917c48 namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: a3d6aa08-3180-4d0f-a5c1-afb02dacdb2d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d09b9} kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:09:29Z" message: 'Created container: switch-graph-d09b9' metadata: creationTimestamp: "2026-07-08T17:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:29Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c05f9a680668c2 namespace: kserve-ci-e2e-test resourceVersion: "27362" uid: 929a7f79-14fb-4b1e-b795-559cf8d5c208 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d09b9} kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:09:29Z" message: Started container switch-graph-d09b9 metadata: creationTimestamp: "2026-07-08T17:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:09:29Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c05f9a68c40683 namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: f215693c-3614-4fc0-b98b-529ee99fb12b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d09b9} kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:25:47Z" message: Stopping container switch-graph-d09b9 metadata: creationTimestamp: "2026-07-08T17:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:25:47Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c0607e2b1a1d10 namespace: kserve-ci-e2e-test resourceVersion: "34228" uid: 4631f29c-06f6-4bc4-934f-ee60445f482a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:25:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d09b9} kind: Pod name: switch-graph-d09b9-bd8cfc589-h94lt namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 30bac7f5-9029-4449-9acd-55ed5acb46cb kind: Event lastTimestamp: "2026-07-08T17:26:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:26:11Z" name: switch-graph-d09b9-bd8cfc589-h94lt.18c0607f0860e808 namespace: kserve-ci-e2e-test resourceVersion: "34478" uid: 79800cca-3d80-4bb3-8b98-007c8f9bfcc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d09b9-bd8cfc589 namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: d12be4a7-f87f-4b89-96f1-6ca755724dd6 kind: Event lastTimestamp: "2026-07-08T17:09:28Z" message: 'Created pod: switch-graph-d09b9-bd8cfc589-h94lt' metadata: creationTimestamp: "2026-07-08T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:28Z" name: switch-graph-d09b9-bd8cfc589.18c05f9a24e97778 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: 7b1f3ba1-cea7-4a5c-baba-f0b584e73b09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27323" uid: 6a99cf98-fccf-4fe4-8d27-92d5e972e102 kind: Event lastTimestamp: "2026-07-08T17:09:28Z" message: Scaled up replica set switch-graph-d09b9-bd8cfc589 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:28Z" name: switch-graph-d09b9.18c05f9a23f735e5 namespace: kserve-ci-e2e-test resourceVersion: "27328" uid: 64214319-8afa-4958-acb5-11f8a8ea9b6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27344" uid: 47411a01-efc7-42df-b6e9-d9c2bcc5afef kind: Event lastTimestamp: "2026-07-08T17:09:28Z" message: InferenceGraph [switch-graph-d09b9] is Ready metadata: creationTimestamp: "2026-07-08T17:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:09:28Z" name: switch-graph-d09b9.18c05f9a29c09b23 namespace: kserve-ci-e2e-test resourceVersion: "27347" uid: dbd8dd20-d588-48bd-b151-5ab63a4b22ba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: ac082471-1537-4aa5-b280-ca26abbece7c kind: Event lastTimestamp: "2026-07-08T17:09:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:09:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:43Z" name: switch-graph-d09b9.18c05f9da27818ba namespace: kserve-ci-e2e-test resourceVersion: "27485" uid: 520ce1f1-8673-42b0-89c3-e795fe60369b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: ac082471-1537-4aa5-b280-ca26abbece7c kind: Event lastTimestamp: "2026-07-08T17:09:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:09:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:43Z" name: switch-graph-d09b9.18c05f9da2e37c83 namespace: kserve-ci-e2e-test resourceVersion: "27489" uid: 155b6b4c-7789-40f1-ac22-13bc3d6dfb15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: ac082471-1537-4aa5-b280-ca26abbece7c kind: Event lastTimestamp: "2026-07-08T17:09:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:58Z" name: switch-graph-d09b9.18c05fa121acaad0 namespace: kserve-ci-e2e-test resourceVersion: "27644" uid: 2e6d8175-6126-4abb-a579-3b1dcc64b421 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:09:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d09b9 namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: ac082471-1537-4aa5-b280-ca26abbece7c kind: Event lastTimestamp: "2026-07-08T17:09:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:09:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:09:58Z" name: switch-graph-d09b9.18c05fa1221696e1 namespace: kserve-ci-e2e-test resourceVersion: "27647" uid: 37455eb0-b995-4439-9291-fe6566d69362 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:47:37.793790Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f6c4d-6dfbf84d47-nk85s to ip-10-0-129-12.ec2.internal metadata: creationTimestamp: "2026-07-08T16:47:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:47:37Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e691150cfdc namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 7c2febdf-8bcd-43d7-ba55-cc7ee05c3a79 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8648d497b5-z8x8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:47:38Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:47:38Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e692ad512dc namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: fab8cebf-37a1-40fb-b55b-41600fb8ca94 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f6c4d} kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:47:38Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" metadata: creationTimestamp: "2026-07-08T16:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:38Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e692c0313f1 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: da107a6d-506d-4260-a0c3-9750a8602ca3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f6c4d} kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:47:40Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" in 2.609s (2.609s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-08T16:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:40Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e69c791146e namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 29ebd390-a695-48f0-bbf1-369ec6b15d4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f6c4d} kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:47:40Z" message: 'Created container: switch-graph-f6c4d' metadata: creationTimestamp: "2026-07-08T16:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:40Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e69cae7b88a namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 38784bf7-3c91-4ff3-85ba-2b69acedccfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f6c4d} kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:47:40Z" message: Started container switch-graph-f6c4d metadata: creationTimestamp: "2026-07-08T16:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:40Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e69cb5bb375 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: fcec0580-706f-47bd-8aa3-4faeb44cf948 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f6c4d} kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:47:47Z" message: Stopping container switch-graph-f6c4d metadata: creationTimestamp: "2026-07-08T16:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:47:47Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e6b6fa636ce namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 4ff248af-4965-474c-932d-308463b44db6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:47:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f6c4d} kind: Pod name: switch-graph-f6c4d-6dfbf84d47-nk85s namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ac9e3c73-6555-4e6a-98a8-15228438e5f3 kind: Event lastTimestamp: "2026-07-08T16:48:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:17Z" name: switch-graph-f6c4d-6dfbf84d47-nk85s.18c05e6c6170125b namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 53eaf92a-cba7-459c-a2a4-de4a240eec53 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-12.ec2.internal source: component: kubelet host: ip-10-0-129-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f6c4d-6dfbf84d47 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: db8ce6a5-38e4-467d-8749-15742ce07891 kind: Event lastTimestamp: "2026-07-08T16:47:37Z" message: 'Created pod: switch-graph-f6c4d-6dfbf84d47-nk85s' metadata: creationTimestamp: "2026-07-08T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:37Z" name: switch-graph-f6c4d-6dfbf84d47.18c05e6910efd5e5 namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: e7acd84b-0ec8-49c5-b5e3-175a0bd38cc9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f6c4d namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: fe253f79-9fec-4636-9209-753c9606a8e5 kind: Event lastTimestamp: "2026-07-08T16:47:37Z" message: Scaled up replica set switch-graph-f6c4d-6dfbf84d47 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:37Z" name: switch-graph-f6c4d.18c05e6910201618 namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: 3a57dc5b-0b70-4454-80b9-bb3bea8c1758 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:47:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f6c4d namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 86c22799-8d07-4006-87bc-984b4ff9e79a kind: Event lastTimestamp: "2026-07-08T16:47:37Z" message: InferenceGraph [switch-graph-f6c4d] is Ready metadata: creationTimestamp: "2026-07-08T16:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:47:37Z" name: switch-graph-f6c4d.18c05e69160312ca namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 0efcea4b-c287-43ac-8b43-f27d4591e203 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34512"