--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:43:51.995413Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-68c6a-6655b859d7-hlhjf to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:43:51Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681cfd8dae74b namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 94492f50-d61b-4c6d-98cf-3e69f8aa58ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:43:52Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681cff2d916c8 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: b065dc51-5bf2-455c-97bf-d2548a78c709 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68c6a} kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681cff446072b namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 951489b6-1b8d-4cca-8681-0f1a1c866abd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68c6a} kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: 'Created container: ensemble-graph-68c6a' metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681cff6e9051a namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: b0481471-7162-4c97-88a6-4f3fb533bf42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68c6a} kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Started container ensemble-graph-68c6a metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681cff75f2df4 namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 977bb02d-f8f1-456a-b4a8-1ffccd3da759 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68c6a} kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: "2026-07-28T16:44:01Z" message: Stopping container ensemble-graph-68c6a metadata: creationTimestamp: "2026-07-28T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:01Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681d22b621ede namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: f8635f35-b1b5-44b0-8367-4bedd14c1b2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68c6a} kind: Pod name: ensemble-graph-68c6a-6655b859d7-hlhjf namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: fee6ca21-44cc-4acd-bcba-4de270914ddd kind: Event lastTimestamp: "2026-07-28T16:44:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:28Z" name: ensemble-graph-68c6a-6655b859d7-hlhjf.18c681d28b1a60a3 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: a54fe540-2230-48f7-a496-1921d2999c40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-68c6a-6655b859d7 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: a83e510f-58c9-453a-858b-2ac8477a0085 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'Created pod: ensemble-graph-68c6a-6655b859d7-hlhjf' metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: ensemble-graph-68c6a-6655b859d7.18c681cfd80678db namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 32fffa7c-2ca6-4367-a0a4-fb63fdb7c6c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20972" uid: 91a93003-9545-44e7-bec2-eb488bb3963c kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Scaled up replica set ensemble-graph-68c6a-6655b859d7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: ensemble-graph-68c6a.18c681cfd62ea74a namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 290783bb-c81c-4b4e-8546-f2186b7dfc55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-68c6a namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: e43f691a-07fc-4637-b655-b43a862232e3 kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: InferenceGraph [ensemble-graph-68c6a] is Ready metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:52Z" name: ensemble-graph-68c6a.18c681cfdff23c85 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 4ad99ab2-9115-4a59-ae58-171680a3b0ab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:45:02.194411Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c3541-95c57758-jf8c6 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c681e0310ab0df namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: a9893e1a-48d6-4e6b-890b-879baf2af2e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c681e04a405b94 namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 4f2d63a5-9620-4aea-a98d-ab6667fd44b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c3541} kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c681e04bac54a0 namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 689b524c-25ab-42e0-b0cb-50294326b584 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c3541} kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'Created container: ensemble-graph-c3541' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c681e04e567bda namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 0fde8a6f-8a60-46f4-a9fc-e353f3534872 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c3541} kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Started container ensemble-graph-c3541 metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c681e04ecbbe10 namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: 28aea0b2-158c-415a-949c-002034ab6d5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c3541} kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: "2026-07-28T16:53:16Z" message: Stopping container ensemble-graph-c3541 metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:16Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c682535be8712b namespace: kserve-ci-e2e-test resourceVersion: "25838" uid: ae43abea-9f6e-416d-892f-ba90ac25a795 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c3541} kind: Pod name: ensemble-graph-c3541-95c57758-jf8c6 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: a38c11aa-4828-4aea-a1dc-390e58724ebe kind: Event lastTimestamp: "2026-07-28T16:53:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:43Z" name: ensemble-graph-c3541-95c57758-jf8c6.18c68253d45f4772 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 648e2c9d-5513-4695-9042-75d76458a754 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c3541-95c57758 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: f3787802-4779-4891-8625-bb23e719a0ad kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'Created pod: ensemble-graph-c3541-95c57758-jf8c6' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541-95c57758.18c681e03071a6fd namespace: kserve-ci-e2e-test resourceVersion: "21966" uid: 56388683-746e-4cbe-a0b6-c2cc3827eb09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21946" uid: aa2104a2-5261-4b1a-85b9-f63f49bf4671 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Scaled up replica set ensemble-graph-c3541-95c57758 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541.18c681e02f8d23c1 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: 7ceb01ab-a5fc-416c-9f85-51da27c7ea5c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: 5f6d78ea-a523-4d0e-9b47-b894644aeb26 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: InferenceGraph [ensemble-graph-c3541] is Ready metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:02Z" name: ensemble-graph-c3541.18c681e036aaf5dd namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: fa154992-9551-45a5-8db7-a5acecf35bda reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: b531366d-5e15-409a-aa52-12bd8bbfa606 kind: Event lastTimestamp: "2026-07-28T16:45:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:47Z" name: ensemble-graph-c3541.18c681e3adf6098b namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: bb398a7e-3387-4c14-9338-cbff0494f82a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: b531366d-5e15-409a-aa52-12bd8bbfa606 kind: Event lastTimestamp: "2026-07-28T16:45:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:47Z" name: ensemble-graph-c3541.18c681e3ae867cdf namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: 92ab767d-a62a-491d-a60a-cad91266f016 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:45:02.182134Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0304f4faf namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: 2d72a9a1-0f5c-4e57-bbbc-95f23e08c9b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-12f5e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e03f1d2637 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 760ceb46-629a-4ec8-bdcf-9061eeddf380 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0a2bae5ec namespace: kserve-ci-e2e-test resourceVersion: "22031" uid: d654b6ef-1e53-4f85-83de-01b8ca68ae98 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0a404840c namespace: kserve-ci-e2e-test resourceVersion: "22033" uid: d2e95ff0-d855-4fea-83df-40fc134ced31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0a75a4c71 namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 551dbb8b-4d1d-4c27-b12e-bfcb29de4719 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0a7ce1475 namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: 621d64ca-b836-41d6-838d-0ff4c3af53d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0a7d82681 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 8a0a8479-f24a-4f5e-9f96-a29305781b4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0aa79404e namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: 2622d5c8-18f7-43f2-813e-eccc1d2d76a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:04Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0aaeb3f64 namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: fec28d29-d6b3-49ed-bb1e-a254d0573013 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:45:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:40Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c681e0cfce59e1 namespace: kserve-ci-e2e-test resourceVersion: "22341" uid: 06e37852-5fdf-494a-be7e-ebd6cfee9f9e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c6826155835ece namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: 78717a04-096d-4083-9b67-1d1952d48c5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8faf142d-7063-4055-880b-dea1c7a5851f kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2.18c682615583d102 namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 953abf54-4d8b-4508-b94e-f5fe3f8ce07d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-12f5e-predictor-79d5b64b64 namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 61ffa0f0-0e4d-4e72-8de8-fb9aff6b91c8 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'Created pod: error-404-isvc-12f5e-predictor-79d5b64b64-5qnk2' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-12f5e-predictor-79d5b64b64.18c681e02f9408a1 namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: fa1cc6e1-a35c-43ad-b428-8766b2ffc348 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: 5e02a118-51fc-40a3-bcd0-252b5cb928e3 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Scaled up replica set error-404-isvc-12f5e-predictor-79d5b64b64 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-12f5e-predictor.18c681e02d7ee5d3 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 84e3d36b-fcde-44cc-afb7-3a01a2a67c31 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 0e29da1d-3c57-41cb-8610-3f4cbec9d7a6 kind: Event lastTimestamp: "2026-07-28T16:45:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:47Z" name: error-404-isvc-12f5e-predictor.18c681e3ac18d820 namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: b9157189-4331-4d58-9ae0-e45558ec1435 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 0e29da1d-3c57-41cb-8610-3f4cbec9d7a6 kind: Event lastTimestamp: "2026-07-28T16:45:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:47Z" name: error-404-isvc-12f5e-predictor.18c681e3ac96907c namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: 79e168e1-6464-4fcb-b5c0-060c915c9436 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:45:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 0e29da1d-3c57-41cb-8610-3f4cbec9d7a6 kind: Event lastTimestamp: "2026-07-28T16:46:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:17Z" name: error-404-isvc-12f5e-predictor.18c681e72b51ac58 namespace: kserve-ci-e2e-test resourceVersion: "22666" uid: 5d0d5347-160e-4a02-bf46-eff82cad11b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:45:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 0e29da1d-3c57-41cb-8610-3f4cbec9d7a6 kind: Event lastTimestamp: "2026-07-28T16:46:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:17Z" name: error-404-isvc-12f5e-predictor.18c681e72bd18d0e namespace: kserve-ci-e2e-test resourceVersion: "22667" uid: ecead420-6e6f-46d2-b3d3-3a4ede7d8758 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 85f9fbe0-32b5-4afa-95bb-31bfcdb5b9fc kind: Event lastTimestamp: "2026-07-28T16:45:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:51Z" name: error-404-isvc-12f5e.18c681e02d91a2c5 namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 9db50931-886d-4eac-90d9-95f57b4d2ebc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 85f9fbe0-32b5-4afa-95bb-31bfcdb5b9fc kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-12f5e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-12f5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-12f5e.18c681e02e9a2eb3 namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: 3a210b5c-8516-4095-b628-513d8f2201be reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 85f9fbe0-32b5-4afa-95bb-31bfcdb5b9fc kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-12f5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-12f5e.18c681e02e9a9286 namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 67f39c6b-c603-4208-b0d1-22392c85118f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "22432" uid: 85f9fbe0-32b5-4afa-95bb-31bfcdb5b9fc kind: Event lastTimestamp: "2026-07-28T16:45:50Z" message: InferenceService [error-404-isvc-12f5e] is Ready metadata: creationTimestamp: "2026-07-28T16:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:50Z" name: error-404-isvc-12f5e.18c681eb8be795ee namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: e226945a-9240-43b4-a8c0-04dccc16bdf1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:53:17.088602Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2717b-predictor-5c5b975b48-gtnpv to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c682536b08b8b4 namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 1937958d-013c-4624-98df-dd69906336cd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c68253843f0ea6 namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: ecee10ae-90e5-41e6-9e26-3a1519427ae0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c68253857b3898 namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: bed4a937-fd4f-48ab-a627-34a1447247a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c682538844f8ca namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 03471532-8825-4a55-bf3a-602312545a26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c6825388bd6309 namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 6c4c56d0-8c0c-414b-baef-f6a698206b30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c6825388c64c67 namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: ac615883-3c98-4597-aebc-f2c3c4923015 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c682538b89eb7a namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: bb7a5dab-4c24-4161-bdce-f2434ae0d240 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c682538bf87637 namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 5e683c5d-b94f-46d3-9108-7bf591d382e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:53:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:54Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c68253c481c1bb namespace: kserve-ci-e2e-test resourceVersion: "26256" uid: 7f71f692-6e14-4bf1-851e-beaa2a78d5c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c68263c356d3e7 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 47a38266-b146-4c5b-8e07-9a2c74e72648 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c68263c3585621 namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 3fef4f28-2d71-4a1c-b6ee-63d9804ed195 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: a6c5545c-58dd-4a4a-b965-cfbb88ac7ba0 kind: Event lastTimestamp: "2026-07-28T16:54:29Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:29Z" name: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv.18c682644cc84433 namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: a0eaef1c-cb2b-41dc-be37-c16098548a31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2717b-predictor-5c5b975b48 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 62699e2f-0f3c-4a71-b8f2-8fac53f89a91 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Created pod: error-404-isvc-2717b-predictor-5c5b975b48-gtnpv' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor-5c5b975b48.18c682536abee072 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 2bec8be7-3bae-46b9-ae1a-439e3ee9815b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: cc2c37d7-92e9-4bb1-ace3-989927368d6d kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Scaled up replica set error-404-isvc-2717b-predictor-5c5b975b48 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b-predictor.18c682536913f4b2 namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 11ee57c7-cd29-4314-be6f-95bcba82c858 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:53:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 4a261793-9472-4254-a9a3-7bdcd2cbffc2 kind: Event lastTimestamp: "2026-07-28T16:53:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:47Z" name: error-404-isvc-2717b-predictor.18c68256e7bb2a26 namespace: kserve-ci-e2e-test resourceVersion: "26201" uid: 997827de-1348-4e7d-95d7-9b4741e0a205 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:53:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 4a261793-9472-4254-a9a3-7bdcd2cbffc2 kind: Event lastTimestamp: "2026-07-28T16:53:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:47Z" name: error-404-isvc-2717b-predictor.18c68256e824de83 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: d5c8a8c0-34ab-4b31-8f92-bf163cd429d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 4a261793-9472-4254-a9a3-7bdcd2cbffc2 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:17Z" name: error-404-isvc-2717b-predictor.18c6825de58b7d79 namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 739895bd-fe9e-404b-9bf4-b4e848ffb891 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 4a261793-9472-4254-a9a3-7bdcd2cbffc2 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:17Z" name: error-404-isvc-2717b-predictor.18c6825de5f3051b namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: 95c0333b-1b77-4cc6-aeb2-14828dba3d4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 9c8bb47b-80ba-408b-a4d1-b8b7f70a36e3 kind: Event lastTimestamp: "2026-07-28T16:54:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:04Z" name: error-404-isvc-2717b.18c68253691b8dd1 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 8e5615b9-4712-4de3-9cfb-d807bfe158d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 9c8bb47b-80ba-408b-a4d1-b8b7f70a36e3 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-2717b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2717b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b.18c68253699fcc1d namespace: kserve-ci-e2e-test resourceVersion: "25917" uid: 41a65159-0d85-40aa-b9f7-130b50678f05 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 9c8bb47b-80ba-408b-a4d1-b8b7f70a36e3 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2717b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-2717b.18c6825369a01b52 namespace: kserve-ci-e2e-test resourceVersion: "25918" uid: 3f507d26-ac89-4c1a-ab81-467d7b8b0859 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 9c8bb47b-80ba-408b-a4d1-b8b7f70a36e3 kind: Event lastTimestamp: "2026-07-28T16:54:04Z" message: InferenceService [error-404-isvc-2717b] is Ready metadata: creationTimestamp: "2026-07-28T16:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:04Z" name: error-404-isvc-2717b.18c6825e7f9a15e4 namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e75c5df5-761b-407a-b295-f9496787c3e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:34:37.222218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2b606-predictor-76cf8477c8-wnb86 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:34:37Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eadcbc90e namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 12441870-16be-43bb-aecf-31ebb30a3416 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2b606-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eb6a29119 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: affed11c-0549-4708-86a4-b9f699953d00 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eeaeaee2b namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: 7c2564b0-7558-4b9a-83d7-c61be2612537 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eec45ca64 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 340fbe74-904d-4fe3-95be-37546a26b050 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eef421087 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 54c73cf9-ccd2-46a9-b495-9a050a6adcc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eefb89fcf namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: c6098ebf-629c-4f62-8600-185c75269060 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814eefc119cf namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 4d158912-4a11-4b10-aca8-0edfdbced84d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814ef28ca6dc namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 03f01342-5bde-476b-bc5d-00b5abe9b87a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814ef3479d37 namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: 5a8d6ce5-cf90-48f6-ad9c-08b97af1df67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:43:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:54Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c6814f00b804aa namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 4f22cc3c-d5fb-45f4-873e-657af62db3ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:51Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c681cfd63aa58a namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 31bc51c8-ffa2-40f5-a61d-9f92671872e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:51Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c681cfd63b1a30 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 141674fc-cdc1-4c48-8aae-e23c0f372383 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6ec7b3f7-6eae-42e8-9e8c-364b14e5eed0 kind: Event lastTimestamp: "2026-07-28T16:43:54Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:54Z" name: error-404-isvc-2b606-predictor-76cf8477c8-wnb86.18c681d07526c842 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 23078bb5-3738-4910-99cb-534be903a765 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2b606-predictor-76cf8477c8 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 65fa7682-b597-4558-a2fd-eeb2f69e63c8 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'Created pod: error-404-isvc-2b606-predictor-76cf8477c8-wnb86' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:37Z" name: error-404-isvc-2b606-predictor-76cf8477c8.18c6814ead12c04f namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: d9c4302e-d06d-458d-b270-a765295210d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 91ba2a83-787c-4bc0-90fb-43a6aa71b717 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: Scaled up replica set error-404-isvc-2b606-predictor-76cf8477c8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:37Z" name: error-404-isvc-2b606-predictor.18c6814ea9d98811 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 8111a647-b748-4c5c-ad5c-736c33378ca6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:34:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 4db98769-d71e-4298-b4cc-23c9a8ac69d8 kind: Event lastTimestamp: "2026-07-28T16:35:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:22Z" name: error-404-isvc-2b606-predictor.18c6815227c123b7 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 63535025-e1d9-4a08-80c6-665fa9c79369 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:34:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 4db98769-d71e-4298-b4cc-23c9a8ac69d8 kind: Event lastTimestamp: "2026-07-28T16:35:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:22Z" name: error-404-isvc-2b606-predictor.18c68152282870b9 namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: 71202424-805e-45ac-a235-d9933b1a949a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 4db98769-d71e-4298-b4cc-23c9a8ac69d8 kind: Event lastTimestamp: "2026-07-28T16:36:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:36:07Z" name: error-404-isvc-2b606-predictor.18c68155a6ebb9cf namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 2bbb423d-747d-4539-bf17-840f481ff2e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 4db98769-d71e-4298-b4cc-23c9a8ac69d8 kind: Event lastTimestamp: "2026-07-28T16:36:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:36:07Z" name: error-404-isvc-2b606-predictor.18c68155a74bc8cc namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: 3fc4d2d9-468e-4946-881c-68f08e3b67b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b606 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 486272ac-90a0-4ffe-aba6-6933726725ab kind: Event lastTimestamp: "2026-07-28T16:35:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:35:24Z" name: error-404-isvc-2b606.18c6814ea974faab namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: d157c24c-224b-41c0-b580-8e7fc64e6c8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b606 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 486272ac-90a0-4ffe-aba6-6933726725ab kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-2b606": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2b606": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:37Z" name: error-404-isvc-2b606.18c6814eaa80b7e3 namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: 993962f6-b47f-476a-8d5c-cdb60c2bd72d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b606 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 486272ac-90a0-4ffe-aba6-6933726725ab kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2b606": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:37Z" name: error-404-isvc-2b606.18c6814eaa81787c namespace: kserve-ci-e2e-test resourceVersion: "16692" uid: fba517cb-7e27-47e3-af0c-0cedae8c420f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2b606 namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 486272ac-90a0-4ffe-aba6-6933726725ab kind: Event lastTimestamp: "2026-07-28T16:35:24Z" message: InferenceService [error-404-isvc-2b606] is Ready metadata: creationTimestamp: "2026-07-28T16:35:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:35:24Z" name: error-404-isvc-2b606.18c68159bb2dbc34 namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: 54f14d59-f9e3-465c-b7c3-65ff24ebeb90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:42:51.928364Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-68c6a-predictor-846757f6bb-2ss5d to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c1dc94811a namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: ed201fc7-7c7f-4312-82a7-742ffa1862f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-68c6a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c1e695330d namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: b275f139-1ee3-4314-b7f1-fc62fcf26e86 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:42:52Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c2195ba701 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: b8d9f0a0-cc28-4fdc-9c14-f3df5c2e4865 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c21a9c286d namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: ad84a46c-86b3-4eb3-a6f0-f244f20bfd47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:53Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c21db0b986 namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: 6425042f-dc5c-4172-b71e-1dcfea8cd326 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:53Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c21e20cf5f namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 136cdfbb-5cdd-44e5-9aec-1b432825f97f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:53Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c21e298e45 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 1c90421d-3e03-4941-af1c-9b75c5a54561 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:53Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c2213c0c34 namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 0009276a-5c8b-47b6-a32c-379c211f5c1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:42:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:53Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c221adb63b namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 3659b709-3c31-44fe-ad16-0e8b28185074 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:43:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:29Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681c230ca55b4 namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 1029f25d-746f-4977-8555-f329e1bb4668 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681d238141162 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: f1244394-fb72-49c3-9330-108b56b3afb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681d2381493df namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 1c4abd65-ce8b-432d-8081-a14c32c82349 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 53558dfa-f244-4567-a33e-02df75873ba1 kind: Event lastTimestamp: "2026-07-28T16:44:04Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:04Z" name: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d.18c681d2b8ed0b5c namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: 218e6072-e49c-431f-9552-15eb4de889c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-68c6a-predictor-846757f6bb namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: fe3f9c3d-cd1b-419f-93c6-7657aac6856f kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: 'Created pod: error-404-isvc-68c6a-predictor-846757f6bb-2ss5d' metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-68c6a-predictor-846757f6bb.18c681c1dc2b69c0 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: c3029138-e14e-4150-afe0-329f2395032e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: b4e48730-4810-44cc-865e-128bba316045 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: Scaled up replica set error-404-isvc-68c6a-predictor-846757f6bb from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-68c6a-predictor.18c681c1da7d1574 namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 1b658df0-ddae-4be7-9f99-425ce243d875 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: fdd3a858-7939-4252-90be-457142c2ef0a kind: Event lastTimestamp: "2026-07-28T16:43:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:36Z" name: error-404-isvc-68c6a-predictor.18c681c55918720e namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 16a47ec1-91dc-4eed-8ac3-53b8b4e68575 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: fdd3a858-7939-4252-90be-457142c2ef0a kind: Event lastTimestamp: "2026-07-28T16:43:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:36Z" name: error-404-isvc-68c6a-predictor.18c681c55982ba1d namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: abae7c83-af31-4fa2-8b1a-31f2263b90f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:43:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: fdd3a858-7939-4252-90be-457142c2ef0a kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: error-404-isvc-68c6a-predictor.18c681c8d842a658 namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: 3bebaee3-15e3-4573-8896-c87624023871 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:43:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: fdd3a858-7939-4252-90be-457142c2ef0a kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-68c6a-predictor.18c681c8d8abe43a namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 58723aa0-b15d-4298-9bea-01ddb1e5436b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 60362182-c720-48b9-9389-d04f5698c4e2 kind: Event lastTimestamp: "2026-07-28T16:43:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:39Z" name: error-404-isvc-68c6a.18c681c1da8ddfe0 namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: be3ad078-520b-4b1c-bbcc-953765bd2ca9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 60362182-c720-48b9-9389-d04f5698c4e2 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-68c6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68c6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-68c6a.18c681c1db4038ee namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 93e416a1-1903-4033-8900-c7abd79ade11 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 60362182-c720-48b9-9389-d04f5698c4e2 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68c6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-68c6a.18c681c1db408808 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 98bae4a1-5e3c-4425-a06c-c413ee03c0cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 60362182-c720-48b9-9389-d04f5698c4e2 kind: Event lastTimestamp: "2026-07-28T16:43:39Z" message: InferenceService [error-404-isvc-68c6a] is Ready metadata: creationTimestamp: "2026-07-28T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:39Z" name: error-404-isvc-68c6a.18c681cceb86c088 namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: dd59ab4f-0dbf-4c6b-b705-ca645529ad6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:33:37.063471Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140ac0e4d66 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: de78a105-a4ef-4594-83d2-c4b6bc91d63b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140c5906a11 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 3c0e9bb8-1e51-4cf2-8718-985465576183 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140c6d095e9 namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: f967b892-f850-42fe-804c-e8d0e89ef9ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140c98bb920 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: 0b06763f-14c0-45b6-9cfc-577bcb26001e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140c9fa1310 namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: 724f5676-6083-4966-bc77-d4c69088bb4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140ca057c41 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 6d53046c-7437-488e-8220-42fa17d5d031 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140cc897cdc namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 63b084cb-192e-45cd-90e5-6dceffcfcefc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68140cd112e69 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: a0439303-7384-4927-8de7-cb1cbbbdcf20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:33:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:34:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:15Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c68141369e51a2 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 7e677fe2-8c45-41e9-8673-cce7bfd12a98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c681c1db66e249 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: 7ff83469-1b1f-41d1-a2e5-616f3ff7d185 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6a14f67c-97b7-4857-a8f1-61bae732ad19 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:51Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv.18c681c1db67411d namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 8adbd054-79aa-458d-b447-f5525b6610bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7ba9d-predictor-59b97b6f68 namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 5ce96ad9-414d-4690-b140-45409b792451 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: 'Created pod: error-404-isvc-7ba9d-predictor-59b97b6f68-jvqvv' metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor-59b97b6f68.18c68140aba15f2e namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: b441cfcc-29e2-4001-9dee-00c340a8c511 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 268d8123-f74a-429e-98ef-6c9ee1176c01 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Scaled up replica set error-404-isvc-7ba9d-predictor-59b97b6f68 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d-predictor.18c68140aa087db4 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 9fdade5e-4762-4717-8745-3e16dc3ba522 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:33:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: ed6efbf5-efff-49c8-990e-788efda51ee9 kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: error-404-isvc-7ba9d-predictor.18c68144289ad209 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 8cd91012-d21b-4205-acc4-153aa3bd7115 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:33:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: ed6efbf5-efff-49c8-990e-788efda51ee9 kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: error-404-isvc-7ba9d-predictor.18c6814428fe8ccf namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: ca88f115-c0db-4081-88cc-b7d47f92e849 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:34:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: ed6efbf5-efff-49c8-990e-788efda51ee9 kind: Event lastTimestamp: "2026-07-28T16:34:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:52Z" name: error-404-isvc-7ba9d-predictor.18c68147a7ad2bba namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 48a1678a-2c0a-4dbc-a277-146a2e02debc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:34:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: ed6efbf5-efff-49c8-990e-788efda51ee9 kind: Event lastTimestamp: "2026-07-28T16:34:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:52Z" name: error-404-isvc-7ba9d-predictor.18c68147a816a921 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 261ec51c-79c5-4cf3-9514-17570e977a36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: bc3aeb60-e958-443a-9a7a-9da4d6ec9f80 kind: Event lastTimestamp: "2026-07-28T16:34:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:25Z" name: error-404-isvc-7ba9d.18c68140aa090cdd namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 6bf6f3d6-794c-4eb6-84b3-4dae3f523f34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: bc3aeb60-e958-443a-9a7a-9da4d6ec9f80 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-7ba9d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7ba9d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d.18c68140aaa179ec namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 662e46ea-b4a7-4af1-85d1-6f043a9c282f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: bc3aeb60-e958-443a-9a7a-9da4d6ec9f80 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7ba9d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:37Z" name: error-404-isvc-7ba9d.18c68140aaa1c97f namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 8534e00d-8c63-4b93-8806-b7ef819d3614 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: bc3aeb60-e958-443a-9a7a-9da4d6ec9f80 kind: Event lastTimestamp: "2026-07-28T16:34:25Z" message: InferenceService [error-404-isvc-7ba9d] is Ready metadata: creationTimestamp: "2026-07-28T16:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:25Z" name: error-404-isvc-7ba9d.18c6814bf2a7bebc namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: e6c0a599-57a0-44df-9a1d-e91b68598f76 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:43:52.032001Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cfdb094a87 namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: fcfbb5ed-9d76-4eac-8a0a-e0ea039e0941 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cff5074539 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 65ea4f66-0065-4159-b019-0a2473e8800f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cff6aa2182 namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: ebf620c7-1b83-4f06-bd54-317a7a87cd9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cff928aea0 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: e67fcd75-5191-43c6-a6e7-f4514a5b9e86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cff997e3e2 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: ed578406-1aaa-466b-a3cc-25c02c5f25f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cff9a226c3 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 78fce625-e4b5-44ce-8245-932c4bda52ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cffc7485d1 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 2423c56c-c25f-4f7c-b67a-d2d8c572c3f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681cffce47701 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: d04c011c-796b-4eb8-bbab-3a180516228b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:44:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:30Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681d072c63942 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 2b40b29c-b90c-413f-98d4-78fd61d70e77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681e02e70a124 namespace: kserve-ci-e2e-test resourceVersion: "21947" uid: 09b6a0c7-a366-452e-b703-774b515fa8e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f8c4522c-7798-426e-a390-5163faea5e7c kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9.18c681e02e7139a6 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 86dc8944-a207-4486-a4f2-7cb840a30845 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-92dc3-predictor-7f65f58dd namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 5640a2c4-855c-47f9-ac4e-7e4b92ba62db kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: 'Created pod: error-404-isvc-92dc3-predictor-7f65f58dd-dz7g9' metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:52Z" name: error-404-isvc-92dc3-predictor-7f65f58dd.18c681cfda32cc34 namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 0e7b4c14-4c0d-4d04-985d-def8acaf9b12 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 8b61915d-ab79-4f5d-ad75-83c6d02fa016 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Scaled up replica set error-404-isvc-92dc3-predictor-7f65f58dd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: error-404-isvc-92dc3-predictor.18c681cfd575a03e namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 2544678e-6d50-4507-acd2-bbac28f25bc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:44:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: baf9786d-0a5a-4769-a7d4-95de5bb7ad95 kind: Event lastTimestamp: "2026-07-28T16:44:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:36Z" name: error-404-isvc-92dc3-predictor.18c681d353f7b8fa namespace: kserve-ci-e2e-test resourceVersion: "21588" uid: 6ff72dee-ec93-4dd6-ad90-0a3e9d46cbfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:44:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: baf9786d-0a5a-4769-a7d4-95de5bb7ad95 kind: Event lastTimestamp: "2026-07-28T16:44:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:37Z" name: error-404-isvc-92dc3-predictor.18c681d354480e7f namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: f2df0de9-a576-4eae-8cf8-65d5e5379538 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: baf9786d-0a5a-4769-a7d4-95de5bb7ad95 kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:52Z" name: error-404-isvc-92dc3-predictor.18c681ddd1996a03 namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 15cc8578-0f0a-46aa-874b-6794c3454dfc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: baf9786d-0a5a-4769-a7d4-95de5bb7ad95 kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:52Z" name: error-404-isvc-92dc3-predictor.18c681ddd1ff89b7 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 301b8f2d-a17b-4da8-824d-c6051d2b2848 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 25121491-5003-4a7d-b986-d6df50a882f5 kind: Event lastTimestamp: "2026-07-28T16:44:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:40Z" name: error-404-isvc-92dc3.18c681cfd578748c namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: a1cbef7f-ec60-467d-b7b5-094b4e6cddc4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 25121491-5003-4a7d-b986-d6df50a882f5 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-92dc3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-92dc3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:51Z" name: error-404-isvc-92dc3.18c681cfd72708a7 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: d3cc1d5a-3959-4af1-b9a5-4ae67c25c151 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 25121491-5003-4a7d-b986-d6df50a882f5 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-92dc3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:51Z" name: error-404-isvc-92dc3.18c681cfd72752b8 namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 436bc64d-0839-4471-8c91-1bdb5faae4ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: 25121491-5003-4a7d-b986-d6df50a882f5 kind: Event lastTimestamp: "2026-07-28T16:44:40Z" message: InferenceService [error-404-isvc-92dc3] is Ready metadata: creationTimestamp: "2026-07-28T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:40Z" name: error-404-isvc-92dc3.18c681db2df32447 namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: aa77ac8c-4b8b-4484-a66a-9f3247e0e47d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:07.064521Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:07Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812bb7b32ae0 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: dd97ca1d-36a9-4dfe-9683-349d4a881fa9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:07Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812bd16a5a6e namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 075e1b21-7b98-4e97-b4b6-3acc857b0255 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812bd2b3f0c5 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 14e3ec0d-eb96-4dca-896d-b679f4f93b2f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" in 12.715s (12.715s including waiting). Image size: 1333940843 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812ec89f0a91 namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 6d15ee44-04c7-49b7-b786-be5f2f04750e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812ecc01a766 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 031a20f6-72e7-415f-879e-b547c29b9ab5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812ecc73cea8 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 710dda26-d4f2-430f-b13b-0e7638c689fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812ecc8291cb namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: c3c5eaf6-381d-4501-84c7-4f6bab15b30b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.282s (2.282s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:22Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812f549329f4 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: a05e68b9-bc7b-4ae7-ba4d-129279a2a6b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:22Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812f57da9e54 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 6d39d8c7-372d-4dff-b5ff-c84aea443e3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:32:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:22Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812f584fd663 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: c222d3e6-9899-4459-ab87-fc0bc96aa746 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:33:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:39Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6812f729a809b namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 86558688-b7c6-424f-88be-169f3e68d4be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:36Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c68140a594657c namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: f346f932-b6d7-44a5-aa09-7fe6384e6ce9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:36Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c68140a5956d9c namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 916b720a-39e0-4484-a0a1-18ff8644c205 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: a22c69c5-9349-401e-ad26-b1937155ce32 kind: Event lastTimestamp: "2026-07-28T16:33:39Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:39Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6.18c6814124bf308f namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 074d1bf3-64fa-47c0-bca6-6392af06688f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2ad5-predictor-c8c4fb8c4 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 2872817b-c88b-4507-9fa6-ccf7bf61051e kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: 'Created pod: error-404-isvc-b2ad5-predictor-c8c4fb8c4-xbvl6' metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:07Z" name: error-404-isvc-b2ad5-predictor-c8c4fb8c4.18c6812bb7494a44 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: df5a76e0-4d67-425b-9c54-1d87e632fdae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 2c4f7ad7-dde2-4739-a985-0ab1b45e83fd kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Scaled up replica set error-404-isvc-b2ad5-predictor-c8c4fb8c4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:07Z" name: error-404-isvc-b2ad5-predictor.18c6812bb4bb5d07 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 80c3ccec-781f-49ad-b156-fec36786da48 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 8312d485-0633-4e11-b35b-2dc9080e06f9 kind: Event lastTimestamp: "2026-07-28T16:33:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:07Z" name: error-404-isvc-b2ad5-predictor.18c6812f33440e7f namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 4bdd9afb-d4b6-411d-955b-17facf51edc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 8312d485-0633-4e11-b35b-2dc9080e06f9 kind: Event lastTimestamp: "2026-07-28T16:33:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:07Z" name: error-404-isvc-b2ad5-predictor.18c6812f33a935e2 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 517e88fb-345e-4e64-a475-3b7de1ef4e18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 8312d485-0633-4e11-b35b-2dc9080e06f9 kind: Event lastTimestamp: "2026-07-28T16:33:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:22Z" name: error-404-isvc-b2ad5-predictor.18c6813d2e9528b6 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: eb875e20-e148-4d4d-93be-089e52f533b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 8312d485-0633-4e11-b35b-2dc9080e06f9 kind: Event lastTimestamp: "2026-07-28T16:33:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:22Z" name: error-404-isvc-b2ad5-predictor.18c6813d2f038c84 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: e7cea52c-d7ce-45f5-9595-b0f2dc9c2c30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2ad5 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 40844f60-a457-4d46-ac40-c91328495810 kind: Event lastTimestamp: "2026-07-28T16:33:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:09Z" name: error-404-isvc-b2ad5.18c6812bb4bdacfe namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: f1cace7d-e16d-40bb-bb61-d3e8ab74ac44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2ad5 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 40844f60-a457-4d46-ac40-c91328495810 kind: Event lastTimestamp: "2026-07-28T16:33:09Z" message: InferenceService [error-404-isvc-b2ad5] is Ready metadata: creationTimestamp: "2026-07-28T16:33:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:09Z" name: error-404-isvc-b2ad5.18c6813a2d758d90 namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: 9c23beca-7165-4082-898b-225f482976f8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:54:27.393793Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26794" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b6e34-predictor-d4bb678f4-x429j to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263c98cc654 namespace: kserve-ci-e2e-test resourceVersion: "26797" uid: 48945a5a-3f17-4ae9-8c80-c85bce2a5210 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26801" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263e33ec3e1 namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: 03b44c41-a478-415a-b7bd-5499e3134a0d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263e4870bea namespace: kserve-ci-e2e-test resourceVersion: "26820" uid: 90b99415-5cc6-4e63-af42-2cce5ac4bf79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263e7f15ee8 namespace: kserve-ci-e2e-test resourceVersion: "26821" uid: c58cbf3c-9991-4192-9e09-d758326bb387 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263e8637a0e namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 60008dd9-f816-4045-bd6d-f5a91d16886a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263e86ea163 namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: f8ae5b03-d0ac-468b-b1da-52c51220cfcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263eb147c36 namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: f2a1c443-e6e3-43cd-85e7-bd575b81ba71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c68263eb90b3df namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: d50ae7e6-c0c6-4f1d-9e2a-e051eb27cb81 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T17:03:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:44Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c6826420d67d38 namespace: kserve-ci-e2e-test resourceVersion: "30939" uid: 7588a3e2-d76b-4059-a4b2-95ade33b325a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T17:03:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:42Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c682e4f148b863 namespace: kserve-ci-e2e-test resourceVersion: "30915" uid: dc69b9a0-99a5-4b9a-a8b2-7234db3c2b8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T17:03:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:42Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c682e4f1496c1e namespace: kserve-ci-e2e-test resourceVersion: "30916" uid: b8fa5bf1-8a27-4e20-992d-40ffbc354048 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 689c2464-57c6-4a45-8ab0-4de108999c77 kind: Event lastTimestamp: "2026-07-28T17:03:44Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:44Z" name: error-404-isvc-b6e34-predictor-d4bb678f4-x429j.18c682e5953c7fad namespace: kserve-ci-e2e-test resourceVersion: "30938" uid: 9413602b-a4cc-486b-a940-c749487828fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b6e34-predictor-d4bb678f4 namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: 50bdd52c-57c1-4ef2-b7c5-8d47ee9076bb kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'Created pod: error-404-isvc-b6e34-predictor-d4bb678f4-x429j' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor-d4bb678f4.18c68263c8e5c30d namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: 4ba536f5-3b94-469b-8d41-75056b4ae299 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: 4b428297-7a3e-46d4-8066-1d965c792e86 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Scaled up replica set error-404-isvc-b6e34-predictor-d4bb678f4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34-predictor.18c68263c77637c5 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: bd268e78-6534-46e3-9810-03045fdd2d7c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 4526630a-2bf5-425a-9146-0bdc808a2154 kind: Event lastTimestamp: "2026-07-28T16:54:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:57Z" name: error-404-isvc-b6e34-predictor.18c6826745e5bdc9 namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: d20c962b-e4b9-429e-813d-cbc9ed5c611e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 4526630a-2bf5-425a-9146-0bdc808a2154 kind: Event lastTimestamp: "2026-07-28T16:54:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:57Z" name: error-404-isvc-b6e34-predictor.18c68267462f6624 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: c1b9ecdd-b1a0-4097-800c-0ac6f9945838 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 4526630a-2bf5-425a-9146-0bdc808a2154 kind: Event lastTimestamp: "2026-07-28T16:55:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:27Z" name: error-404-isvc-b6e34-predictor.18c6826e43ce9b86 namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: c52b5a85-c87d-4a9c-92cc-7f27b90f30ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 4526630a-2bf5-425a-9146-0bdc808a2154 kind: Event lastTimestamp: "2026-07-28T16:55:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:27Z" name: error-404-isvc-b6e34-predictor.18c6826e44390224 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 9a97d072-1294-4735-9cc6-a0e0d8f76a5d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 24b1f237-5c76-4ef9-bbfa-15944031c99d kind: Event lastTimestamp: "2026-07-28T16:55:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:14Z" name: error-404-isvc-b6e34.18c68263c76fa784 namespace: kserve-ci-e2e-test resourceVersion: "27263" uid: aa1bb793-8347-4c27-8d39-b39afbaf66ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 24b1f237-5c76-4ef9-bbfa-15944031c99d kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-b6e34": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b6e34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34.18c68263c7f15b5a namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: 770ca922-57bf-4bd4-8ace-4f3bdbfab391 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 24b1f237-5c76-4ef9-bbfa-15944031c99d kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b6e34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:27Z" name: error-404-isvc-b6e34.18c68263c7f1968f namespace: kserve-ci-e2e-test resourceVersion: "26792" uid: 639a9c41-a8f6-45e3-9b5b-8c5e2a181b93 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "27260" uid: 24b1f237-5c76-4ef9-bbfa-15944031c99d kind: Event lastTimestamp: "2026-07-28T16:55:14Z" message: InferenceService [error-404-isvc-b6e34] is Ready metadata: creationTimestamp: "2026-07-28T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:14Z" name: error-404-isvc-b6e34.18c6826edb7d5feb namespace: kserve-ci-e2e-test resourceVersion: "27261" uid: b01c230f-840a-4d64-9cb9-27cfdf986c16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:54:16.969342Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c682615c3454d0 namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 94459fd3-8c31-40b2-9802-c8bd7c6542e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b9a8c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c68261696f06ce namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 8498938e-f787-49ea-9214-6063dc2256dd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:54:17Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c682619969694b namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: 7efd8b49-3bd7-48eb-a7ad-cd7e9ffdc300 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:18Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c682619aa83515 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: dd4d223b-3322-4c50-bddd-31799f0b0fe2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:18Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c682619d959bc0 namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: ba320f75-1900-47ae-868b-fd93dba33476 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:18Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c682619e0a85a7 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 15b641d3-37c9-42c5-a661-c62c265f191c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:18Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c682619e1369fb namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 15532a39-d54b-4de5-9389-d46662e25b0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:18Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c68261a0c83c70 namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: 1913e666-3d7b-470f-953f-f3159b49ef99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:18Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c68261a13ac369 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 162da0dc-2da6-4e55-993b-a055cefefb3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T16:54:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:55Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c6826205de4c60 namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: a3aa2182-f272-4acb-bf1d-a116759213c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T17:11:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:11:36Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c683535e79adee namespace: kserve-ci-e2e-test resourceVersion: "34202" uid: 83e0ac64-678c-4342-95ac-d39c11431927 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e85b47f1-f9d8-46f3-8d95-ae9e1c2997b2 kind: Event lastTimestamp: "2026-07-28T17:11:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:11:36Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8.18c683535e7a8e11 namespace: kserve-ci-e2e-test resourceVersion: "34203" uid: 3fc459c4-a289-4b30-8562-2c084b12c8d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b9a8c-predictor-57b5cbbd5b namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 767355a0-9e09-4fbb-87b9-d35b657e8726 kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: 'Created pod: error-404-isvc-b9a8c-predictor-57b5cbbd5b-cm5g8' metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-b9a8c-predictor-57b5cbbd5b.18c682615bccf21f namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: d337c7eb-ad4b-48e1-8466-08dfbce7d4fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26520" uid: f310bde9-6d70-45a8-9e46-56957567fd1d kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Scaled up replica set error-404-isvc-b9a8c-predictor-57b5cbbd5b from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-b9a8c-predictor.18c682615a406190 namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: f46c6924-aadf-4bbd-b35d-63e020e87810 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26521" uid: d45ac140-3e0d-4848-9597-5885c8f688f3 kind: Event lastTimestamp: "2026-07-28T16:54:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:46Z" name: error-404-isvc-b9a8c-predictor.18c68264d8e9285a namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: eb08e6c5-5fb2-4272-8b81-c379d8b17867 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26521" uid: d45ac140-3e0d-4848-9597-5885c8f688f3 kind: Event lastTimestamp: "2026-07-28T16:54:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:46Z" name: error-404-isvc-b9a8c-predictor.18c68264d94fab40 namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: ac9fb606-14b0-49d3-81b3-ea7f6dad8761 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:55:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: d45ac140-3e0d-4848-9597-5885c8f688f3 kind: Event lastTimestamp: "2026-07-28T16:55:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:32Z" name: error-404-isvc-b9a8c-predictor.18c6826bd6f1b7f0 namespace: kserve-ci-e2e-test resourceVersion: "27481" uid: cd9f7567-ab10-493e-8542-e8607273e9e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:55:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: d45ac140-3e0d-4848-9597-5885c8f688f3 kind: Event lastTimestamp: "2026-07-28T16:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:32Z" name: error-404-isvc-b9a8c-predictor.18c6826bd74efc63 namespace: kserve-ci-e2e-test resourceVersion: "27482" uid: 7f84bf5e-5ee7-45db-9c4e-d97fbbe95749 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: a19b5c9f-4a38-4884-a1f3-5030e7f4db9b kind: Event lastTimestamp: "2026-07-28T16:55:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:05Z" name: error-404-isvc-b9a8c.18c682615a3ebe5b namespace: kserve-ci-e2e-test resourceVersion: "27189" uid: 8a77953f-f342-4a6b-ba26-3668dd96123a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: a19b5c9f-4a38-4884-a1f3-5030e7f4db9b kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-b9a8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b9a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-b9a8c.18c682615acb1f11 namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 7620cef6-95af-4238-baa0-6714b521251f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: a19b5c9f-4a38-4884-a1f3-5030e7f4db9b kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b9a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:16Z" name: error-404-isvc-b9a8c.18c682615acb5852 namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 34168f5a-2763-47b6-8d06-82fc0789e5ba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "27187" uid: a19b5c9f-4a38-4884-a1f3-5030e7f4db9b kind: Event lastTimestamp: "2026-07-28T16:55:05Z" message: InferenceService [error-404-isvc-b9a8c] is Ready metadata: creationTimestamp: "2026-07-28T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:05Z" name: error-404-isvc-b9a8c.18c6826cc087b31f namespace: kserve-ci-e2e-test resourceVersion: "27188" uid: e34c6b30-5b1f-4d8d-802c-09369957718f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:44:02.212796Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3541-predictor-67c685758b-jbrfq to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d239dbcec8 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 0ab29824-87a6-4370-a136-c2bb1f7dffcd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d25351f867 namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 45ce5c97-dc5a-4f38-9b84-c9f5b833f290 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d2549f6cac namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 32789ff5-2892-49e6-a0a0-0f2e3793e9ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d257411324 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 3ab80a02-93ea-4f3f-af44-7f6a2ff37332 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d257b2b61e namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: 0197f492-df2b-4c9f-90e0-fe7d16e30ffe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d257bbdc5d namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 5d9e7dd8-8303-4826-90c0-0ee8028b8d2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d25a768937 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 440c07fb-129d-46e6-92e6-cdc3e51702c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d25ae7c842 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 89f29a37-c8b0-4761-8657-9a35427cdf8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:53:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:19Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c681d28d6f3958 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: a72b0be1-df2f-480c-96c0-96bf4c924647 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c6825369419a6b namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 7c37b7e0-a119-4316-8f4b-274f4135e545 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c682536942e10f namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: 7019f281-4721-4bc9-b430-52f2d8c78e0b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3541-predictor-67c685758b-jbrfq namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b2edb5f1-1799-4f12-af81-4659496e8837 kind: Event lastTimestamp: "2026-07-28T16:53:19Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:53:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:19Z" name: error-404-isvc-c3541-predictor-67c685758b-jbrfq.18c6825401d4be52 namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: 922a2ced-f128-4007-8931-bd78b71f0844 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3541-predictor-67c685758b namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: d55dad70-516c-41ec-9632-3af83ebd884e kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Created pod: error-404-isvc-c3541-predictor-67c685758b-jbrfq' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor-67c685758b.18c681d23956c925 namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 0666607e-1dae-402a-91da-4cc2463988be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 55eb9634-d057-4c55-b913-f6da9941189e kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Scaled up replica set error-404-isvc-c3541-predictor-67c685758b from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541-predictor.18c681d237a5670c namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 1470f7ab-a4c5-41d8-b8c2-4bd58603b4d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 0c235756-42a5-470e-bc4f-ca618d5fa9ca kind: Event lastTimestamp: "2026-07-28T16:44:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:32Z" name: error-404-isvc-c3541-predictor.18c681d5b63f9cd4 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: f286a1a8-4ff0-432f-9616-94f62b90ff7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 0c235756-42a5-470e-bc4f-ca618d5fa9ca kind: Event lastTimestamp: "2026-07-28T16:44:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:32Z" name: error-404-isvc-c3541-predictor.18c681d5b6ab401f namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: cec5733e-14c9-4834-80ad-17afd8de21c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 0c235756-42a5-470e-bc4f-ca618d5fa9ca kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-c3541-predictor.18c681dcb456b45d namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: db462a3f-b03b-4d4e-832a-dee597571d23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 0c235756-42a5-470e-bc4f-ca618d5fa9ca kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: error-404-isvc-c3541-predictor.18c681dcb4a44803 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: e7a1949b-708f-4ef4-9451-3a1730226f57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 967c59d6-7e47-4b5f-814c-5590d05f2d93 kind: Event lastTimestamp: "2026-07-28T16:44:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:49Z" name: error-404-isvc-c3541.18c681d237a7f3ce namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: 9f13a820-1671-4a0f-b63c-58d10871d2b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 967c59d6-7e47-4b5f-814c-5590d05f2d93 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3541": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3541": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541.18c681d23827d732 namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: aac19dcd-d559-4085-94db-f8c503876c7d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 967c59d6-7e47-4b5f-814c-5590d05f2d93 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3541": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:02Z" name: error-404-isvc-c3541.18c681d238283f43 namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: fd3e24cd-6d29-473a-b8fa-10cdab99e3c7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: 967c59d6-7e47-4b5f-814c-5590d05f2d93 kind: Event lastTimestamp: "2026-07-28T16:44:49Z" message: InferenceService [error-404-isvc-c3541] is Ready metadata: creationTimestamp: "2026-07-28T16:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:49Z" name: error-404-isvc-c3541.18c681dd48f21d39 namespace: kserve-ci-e2e-test resourceVersion: "21716" uid: aa1f8d18-77a0-4a68-acda-edbd5a303e2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:06.972437Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 to ip-10-0-140-22.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:06Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812bb2361193 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: b28d5503-cfb3-4187-8d7b-37f4512f475c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812bbc311090 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: ecd24f93-3cf5-4448-bf5e-b2762c66e57d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812bf06aee3b namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: 42915d33-ce8e-497f-ad14-2c9800a53b4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812bf21344fa namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 5f7be81f-9fb2-4a67-8e63-53a8edd05249 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812bf55f4fb4 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: ca110b94-83f7-4a23-8b17-d8cc54a07578 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812bf5d88c32 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 493738c7-b033-49b7-ae0c-b2a82244377d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T16:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6812ce1e496d5 namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: c3a45be9-4885-40cc-95a7-aea00f0b6d5a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 24.111s (24.111s including waiting). Image size: 1555831221 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c681327f0ef8a5 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 6a63324c-0ed1-49cd-b911-8c9763085d9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c68132837dba05 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: 493bc554-72f5-4495-8eaf-2af148560f88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c681328437247f namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 95cba27d-5b1d-46da-91fa-3c929c64d640 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c68132844592a6 namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 180fb37d-de72-43be-816a-7e6027e5dbec reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.208s (3.208s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:39Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c681334383b388 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 4234a55e-be1b-4325-a8bd-0f9b3c6b6620 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:39Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c681334acc8301 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 9926d715-fe67-452c-8579-c475f1ca3599 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:32:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:39Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c681334bfcdd92 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 2954b200-2ee3-4963-b9de-ef8665c30e89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c68133acd3153f namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: d43d9409-10d7-487f-80c7-225328ac80bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6814ea1605c37 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: b5602d64-8caa-4567-8fb9-2c5582789b3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6814ea1610a1b namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 1def4f37-83d2-4d75-9e8a-3c355337ce30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: ba8b1a86-7d9a-476f-89ec-f4b4d99786a6 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986.18c6814eaf313323 namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: afe990d6-c161-4eee-9cd9-15d90c9736a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6d55f5d589 namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: f5d31a08-fc20-4383-845a-30621f7a5ffb kind: Event lastTimestamp: "2026-07-28T16:32:06Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6d55f5d589-2n986' metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:06Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589.18c6812bb1cff8c2 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 4ca12caa-e1da-4510-8019-129ae4a44d3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 5ee51e11-cf7d-44cf-b8b6-3f9e15991b75 kind: Event lastTimestamp: "2026-07-28T16:32:06Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6d55f5d589 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:06Z" name: isvc-sklearn-graph-1-predictor.18c6812bae333773 namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 62cd93d1-edc5-41eb-ac63-190d2a64018a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: 393eeb01-b117-4c6e-9119-737d97486caf kind: Event lastTimestamp: "2026-07-28T16:33:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:21Z" name: isvc-sklearn-graph-1-predictor.18c6812f2cdecf6b namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: b8bc2586-86a6-4f54-ad3d-dd1c559d4b1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: 393eeb01-b117-4c6e-9119-737d97486caf kind: Event lastTimestamp: "2026-07-28T16:33:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:21Z" name: isvc-sklearn-graph-1-predictor.18c6812f2d55767e namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 4f502353-af20-4d98-aab3-2c8cf05aa19e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 393eeb01-b117-4c6e-9119-737d97486caf kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: isvc-sklearn-graph-1-predictor.18c68140a7650c36 namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: fed96ff5-e795-46fe-a5b9-2d07e94a5bdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 393eeb01-b117-4c6e-9119-737d97486caf kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: isvc-sklearn-graph-1-predictor.18c68140a7d5462e namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: 89fb0eec-64e7-4019-91ec-86243705d650 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 9371f52a-9ada-44d1-9491-2191a7eb9257 kind: Event lastTimestamp: "2026-07-28T16:33:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:57Z" name: isvc-sklearn-graph-1.18c6812bae349fe8 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: bd479209-b989-44a8-a418-d6cc9213133b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 9371f52a-9ada-44d1-9491-2191a7eb9257 kind: Event lastTimestamp: "2026-07-28T16:33:57Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T16:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:57Z" name: isvc-sklearn-graph-1.18c6814563face19 namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: f56f287f-b481-475c-bcc2-0ed213df3b95 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:07.273950Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:07Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812bc42ed374 namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 34159421-f198-40c6-9cbc-1efc1c2e9be2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812bce5cb7f3 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 82dbf70d-02b0-4100-a92c-25b0e9bd0fdd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:08Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812c04a0f256 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 4c391147-d1c1-48a3-baa8-dfe208d8f3b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" metadata: creationTimestamp: "2026-07-28T16:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:08Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812c068a808e namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: e3561ea5-7f98-49f0-b3b6-a4711b01aab6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" in 11.753s (11.753s including waiting). Image size: 300915674 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812ec3159d46 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 8516ee77-10ff-405c-8a19-20b05261c888 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812ec5d2052d namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 7b05f9c8-431c-4864-a0b9-c06ae8f6b42e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T16:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:20Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812ec6474e33 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 6da2a1e8-cea5-434e-aed0-cf48883dc2c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:25Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T16:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:25Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6812fea50cabd namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 0f5a2fd6-cc70-406b-b6fc-4bea246dd459 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:31Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 6.727s (6.727s including waiting). Image size: 1555831221 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c681317b48ed8c namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: d64bd814-84fc-4188-ad6b-c2d65579896d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c681317e76a627 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: fafa5529-29bd-4144-a28a-9cdbfcb9d209 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c681317ef58c12 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: aa86faf1-0532-4c0e-9296-70ea1290a55b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c681317efe3ed6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: a112098b-2a6f-4d6c-b78e-c974407997a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c68131832c81c7 namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: 74b451ce-760f-419b-baea-2d0dbe7b57d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:32:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6813183a4e515 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 51a5a73e-d25a-4d8d-b82d-26fbfca0203d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T16:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c681318db3cfbf namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: a988f4c2-4928-41fa-94a7-747c0cbed7cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:34:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:36Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6814e9dc6f383 namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: b52aa731-5436-41cc-a127-9fdc702b90b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:34:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:36Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6814e9dc76589 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 181ea8bb-dae5-474f-b1a2-ed9c488e6c29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: e8a1a80a-9dcb-4596-be10-bf900623d462 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb.18c6814ee42e0c69 namespace: kserve-ci-e2e-test resourceVersion: "16735" uid: 9628e20b-66aa-4904-89bc-4148ffa5d085 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7d44765f95 namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 64e00cc9-d82b-47b0-bb5a-e0aa1954dc25 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7d44765f95-z6fkb' metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:07Z" name: isvc-sklearn-graph-2-predictor-7d44765f95.18c6812bc3cb158c namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: 85659104-b2e1-4246-83a9-bb098eafc4a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: a0c69a14-1cbb-4a41-a226-4768d35e8851 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7d44765f95 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:07Z" name: isvc-sklearn-graph-2-predictor.18c6812bc0bcabcd namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 484bed1e-2b8f-4503-836d-f5aa22e10ea5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 8818f96e-e76d-4227-98c3-60a402403d8b kind: Event lastTimestamp: "2026-07-28T16:33:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:07Z" name: isvc-sklearn-graph-2-predictor.18c6812f3f785568 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 12abb294-bb2f-4ce7-88e3-d0c9e2be711d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 8818f96e-e76d-4227-98c3-60a402403d8b kind: Event lastTimestamp: "2026-07-28T16:33:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:07Z" name: isvc-sklearn-graph-2-predictor.18c6812f3fe0222f namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 2accae49-9544-444d-a3d3-979eeb2cd465 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:33:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 8818f96e-e76d-4227-98c3-60a402403d8b kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: isvc-sklearn-graph-2-predictor.18c6813d3afe72bd namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: 7c2b2027-9508-4888-b691-388855a69e60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:33:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 8818f96e-e76d-4227-98c3-60a402403d8b kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: isvc-sklearn-graph-2-predictor.18c6813d3b74c386 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: be7fad43-d007-4825-abe7-cb69123a1053 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: e6c3c07d-42db-4991-bb9a-c02e86a4dd95 kind: Event lastTimestamp: "2026-07-28T16:33:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:48Z" name: isvc-sklearn-graph-2.18c6812bc0bd2b2d namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: eccc3dc7-970f-4f96-ab48-5d41dc28446a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: e6c3c07d-42db-4991-bb9a-c02e86a4dd95 kind: Event lastTimestamp: "2026-07-28T16:33:48Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:48Z" name: isvc-sklearn-graph-2.18c6814344f4e70b namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 0a64c18b-63ba-461a-b439-b06b5cfb46d6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:07.173955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-lf2gm to ip-10-0-140-22.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6812bbe390d60 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 4551b17a-568a-4055-bee6-b6a154c86f2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6812bd81adc11 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 883e7877-fc3b-4d3d-9459-cde365888a9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6812bd97d4555 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 66f26e5e-1f6a-47b1-8bfc-59979c7ceb12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6812bdca49d2f namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 009fa23d-28ce-431e-b983-0e8769dd8c7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6812bdd1a3200 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 062c68cd-42b5-4cce-9c53-fd6f9c0c82fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:12Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T16:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6812ce1ff6446 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: e2cc3b24-4a56-43a1-a9a2-0637e26ff4ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.364s (24.364s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c681328e3c4bcc namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: d6feefdc-4e7b-4a6d-afb5-cb6c14a40937 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c681329250898b namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 9fa21157-aa1d-4980-b226-a153ea2c20bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6813294965c5a namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 68936071-47d0-4cac-a1e8-17d2bd436a73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6813294a4b260 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 13070a17-1f08-4b1b-b5f5-97ba08786f3b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.942s (2.942s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6813344052db9 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 1c66744c-60ca-4e21-9d91-07ab5b8723bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c681334acd33b4 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: aeda4036-5c7a-4264-be2b-5fbf326300fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:32:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c681334bfcdd56 namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: c7a22e40-7a95-4397-9871-c49587ba8ad2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T16:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6813370ceea05 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 6605fb32-4b21-4894-99c0-349ee8315043 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6814ea5113bc6 namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: 071d75fd-6a0a-4cbb-93b1-cfbb5f050f96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3f71a3fc-9d45-415f-8c36-cf045092d0d9 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-lf2gm.18c6814ea511fcde namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 0c600022-6ff5-4d1f-be48-26eebca7eb6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-22.ec2.internal source: component: kubelet host: ip-10-0-140-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: c9f8759b-65c0-484d-8779-b89f78068d72 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-lf2gm' metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c6812bbdcb65c3 namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: abbf8dc2-7197-4ed3-84f9-a542eb62be6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: ac42792d-79ec-434e-b7b1-6202a955f5c8 kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:07Z" name: isvc-xgboost-graph-predictor.18c6812bbac9a977 namespace: kserve-ci-e2e-test resourceVersion: "14856" uid: c5ebf2c7-af59-4cf8-a09f-6fe7aefbdfb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d4a8540a-fe22-413a-b886-63c6092e8285 kind: Event lastTimestamp: "2026-07-28T16:33:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:22Z" name: isvc-xgboost-graph-predictor.18c6812f39565d48 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: af01f13b-3352-4a4b-9b96-d6bab918a746 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d4a8540a-fe22-413a-b886-63c6092e8285 kind: Event lastTimestamp: "2026-07-28T16:33:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:22Z" name: isvc-xgboost-graph-predictor.18c6812f39c30ce8 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 0aba49c5-c01a-454c-a569-a80a9b9bb868 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:33:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: d4a8540a-fe22-413a-b886-63c6092e8285 kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: isvc-xgboost-graph-predictor.18c68139b6004d8e namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 5678832e-1bb5-40d2-b401-95121cacfd2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:33:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: d4a8540a-fe22-413a-b886-63c6092e8285 kind: Event lastTimestamp: "2026-07-28T16:34:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:22Z" name: isvc-xgboost-graph-predictor.18c68139b666cff1 namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 70efb704-3821-4037-ac4c-bac6d0e98c70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: a20fe8fe-da6f-4fd6-bf92-0a70202ea693 kind: Event lastTimestamp: "2026-07-28T16:33:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:46Z" name: isvc-xgboost-graph.18c6812bbacc324d namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: be453c47-d13e-4a5a-af8b-2d8b97b31fb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: a20fe8fe-da6f-4fd6-bf92-0a70202ea693 kind: Event lastTimestamp: "2026-07-28T16:33:46Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T16:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:46Z" name: isvc-xgboost-graph.18c68142d4ac77f5 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 586445f4-96f0-450b-a0c7-d2eaac78e5ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:34:26.721796Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-c5b4df566-hptsh to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:34:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:34:26Z" name: model-chainer-c5b4df566-hptsh.18c6814c3bec0c4e namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: abbb38a9-faa9-42ca-8136-db2b53e07c2f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: "2026-07-28T16:34:27Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:34:27Z" name: model-chainer-c5b4df566-hptsh.18c6814c5549bea1 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 650ddd40-a822-47b7-bb9e-a0268f4de7a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: "2026-07-28T16:34:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:27Z" name: model-chainer-c5b4df566-hptsh.18c6814c5675b346 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 9fc42388-6c92-479d-acc3-607356ef6615 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: "2026-07-28T16:34:27Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:27Z" name: model-chainer-c5b4df566-hptsh.18c6814c597db698 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: e2a57e9e-5264-42da-89aa-8179e744f5eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: "2026-07-28T16:34:27Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T16:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:27Z" name: model-chainer-c5b4df566-hptsh.18c6814c59efec2f namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: 3ac3b532-f07a-4585-a77e-1b9a1c3b1122 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: "2026-07-28T16:34:36Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T16:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:36Z" name: model-chainer-c5b4df566-hptsh.18c6814e9619f782 namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: a573543f-814d-40fa-83d4-a8b52b0dbecf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hptsh namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 0bfe2578-34f0-4b86-8d74-5045ab27a60f kind: Event lastTimestamp: "2026-07-28T16:35:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:35:03Z" name: model-chainer-c5b4df566-hptsh.18c6814efe5d3362 namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: bb542fad-4e51-47f7-ba73-b844cf8804dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-c5b4df566 namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 554b35f5-8deb-410c-a965-041cd2e7c282 kind: Event lastTimestamp: "2026-07-28T16:34:26Z" message: 'Created pod: model-chainer-c5b4df566-hptsh' metadata: creationTimestamp: "2026-07-28T16:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:26Z" name: model-chainer-c5b4df566.18c6814c3b7b3743 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 601c66a0-10ce-4dbd-89ca-1f2835029059 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: 1a4b84f5-d8cc-4c35-a95e-43bfa38cedbd kind: Event lastTimestamp: "2026-07-28T16:34:26Z" message: Scaled up replica set model-chainer-c5b4df566 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:26Z" name: model-chainer.18c6814c3a965fa2 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: 7ce3a086-a066-4fcc-a824-4abdb3631844 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e0afcaf9-e796-4766-9fdf-33d42628ce31 kind: Event lastTimestamp: "2026-07-28T16:34:26Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T16:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:26Z" name: model-chainer.18c6814c3fb0e8eb namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 1c230ff5-f218-4fa3-9966-73483c3fdc7c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:46:02.171654Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-12f5e-77974c8875-46cwl to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:46:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:46:02Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c681ee27f6bc7d namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 6eec064c-f7ea-4725-92c1-33fa729b3285 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:46:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-12f5e-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:46:02Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c681ee381eb9db namespace: kserve-ci-e2e-test resourceVersion: "22549" uid: e0913c3b-70d1-48ce-a12c-48a3f2097f9a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:46:03Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:46:03Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c681ee64be4da4 namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: 08e94d01-95f3-4d05-bd74-72ea80f87920 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12f5e} kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:46:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:46:03Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c681ee660c9f92 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 4067c793-9e68-4b43-93e0-93ea2be60a64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12f5e} kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:46:03Z" message: 'Created container: sequence-graph-12f5e' metadata: creationTimestamp: "2026-07-28T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:46:03Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c681ee6884bb90 namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: f89be068-e0c6-4bb3-91e0-d9e7ff73d9c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12f5e} kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:46:03Z" message: Started container sequence-graph-12f5e metadata: creationTimestamp: "2026-07-28T16:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:46:03Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c681ee68fed3d5 namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: 8c85f87c-8ebd-4ddd-884b-179ed47d9c2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12f5e} kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Stopping container sequence-graph-12f5e metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:16Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c682614c124293 namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: c4e39ab5-419e-4f42-ad80-c5881c17b1ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12f5e} kind: Pod name: sequence-graph-12f5e-77974c8875-46cwl namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 022135ec-1bbe-4c18-84a9-3643ca0edf64 kind: Event lastTimestamp: "2026-07-28T16:54:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:45Z" name: sequence-graph-12f5e-77974c8875-46cwl.18c68262159dee05 namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: 5b0d5be5-4570-4cdb-9fbd-7a3de4b5cc09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-12f5e-77974c8875 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: a15c5ac2-3a19-490f-8d89-39646401a71a kind: Event lastTimestamp: "2026-07-28T16:46:02Z" message: 'Created pod: sequence-graph-12f5e-77974c8875-46cwl' metadata: creationTimestamp: "2026-07-28T16:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:02Z" name: sequence-graph-12f5e-77974c8875.18c681ee2781bb76 namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: edbf3fd9-c94c-44f8-bd05-5e125b70507e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-12f5e namespace: kserve-ci-e2e-test resourceVersion: "22519" uid: 29becec7-39a2-4c5d-85e4-8783b145dc3a kind: Event lastTimestamp: "2026-07-28T16:46:02Z" message: Scaled up replica set sequence-graph-12f5e-77974c8875 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:02Z" name: sequence-graph-12f5e.18c681ee2669efd5 namespace: kserve-ci-e2e-test resourceVersion: "22522" uid: 874780e6-3b57-4f58-ac54-1e9752bae186 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-12f5e namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 47b6b196-36e9-44cb-ae49-c4263b829022 kind: Event lastTimestamp: "2026-07-28T16:46:02Z" message: InferenceGraph [sequence-graph-12f5e] is Ready metadata: creationTimestamp: "2026-07-28T16:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:46:02Z" name: sequence-graph-12f5e.18c681ee2be77c99 namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: b6a4504c-03b8-45b0-b63a-db6638664c27 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-12f5e namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: 7e8219fa-9383-4fdc-82dd-3f08ecb65b3b kind: Event lastTimestamp: "2026-07-28T16:46:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:47Z" name: sequence-graph-12f5e.18c681f1a529f18e namespace: kserve-ci-e2e-test resourceVersion: "22883" uid: befc483c-6e67-4a73-9808-feadeaf53881 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-12f5e namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: 7e8219fa-9383-4fdc-82dd-3f08ecb65b3b kind: Event lastTimestamp: "2026-07-28T16:46:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:47Z" name: sequence-graph-12f5e.18c681f1a5a3ba46 namespace: kserve-ci-e2e-test resourceVersion: "22884" uid: d65978d3-4a7d-4a89-9b7a-54ef35746340 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:35:37.025359Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2b606-679788d849-5mxsm to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:35:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:35:37Z" name: sequence-graph-2b606-679788d849-5mxsm.18c6815c9a574b39 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: 3e2cac9e-f2cc-43a3-96aa-be7819e736b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:35:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-2b606-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:35:37Z" name: sequence-graph-2b606-679788d849-5mxsm.18c6815ca4f2348a namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 6824a158-7a36-4fba-af3b-a2c710b9e967 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:38Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:35:38Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:35:38Z" name: sequence-graph-2b606-679788d849-5mxsm.18c6815cd72dd6c5 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 3ef985b2-a83e-495f-8962-75ad1acb4a3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2b606} kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:35:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:35:38Z" name: sequence-graph-2b606-679788d849-5mxsm.18c6815cd8557cd7 namespace: kserve-ci-e2e-test resourceVersion: "17298" uid: 2b272c26-2ac2-404c-b355-00f485df5bb9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2b606} kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:35:38Z" message: 'Created container: sequence-graph-2b606' metadata: creationTimestamp: "2026-07-28T16:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:35:38Z" name: sequence-graph-2b606-679788d849-5mxsm.18c6815cdbcc1615 namespace: kserve-ci-e2e-test resourceVersion: "17299" uid: e0975cc7-0fdd-40fa-a404-c3f692284657 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2b606} kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:35:38Z" message: Started container sequence-graph-2b606 metadata: creationTimestamp: "2026-07-28T16:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:35:38Z" name: sequence-graph-2b606-679788d849-5mxsm.18c6815cdc3bc0ce namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: 86b94333-e39c-462f-bdd6-7f8b30b463d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2b606} kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Stopping container sequence-graph-2b606 metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:51Z" name: sequence-graph-2b606-679788d849-5mxsm.18c681cfc50ee3d3 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: f69474fd-f982-4d88-acc9-72c66b1369d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2b606} kind: Pod name: sequence-graph-2b606-679788d849-5mxsm namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: c22927ab-2190-4388-b85f-4b8214bad77a kind: Event lastTimestamp: "2026-07-28T16:44:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:19Z" name: sequence-graph-2b606-679788d849-5mxsm.18c681d082a03136 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 504abe62-c63c-4cab-9dcc-bf2fec265673 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2b606-679788d849 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 95e5d2e9-0e64-4695-9a78-e3f1dabfb3ae kind: Event lastTimestamp: "2026-07-28T16:35:37Z" message: 'Created pod: sequence-graph-2b606-679788d849-5mxsm' metadata: creationTimestamp: "2026-07-28T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:37Z" name: sequence-graph-2b606-679788d849.18c6815c99f341b2 namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: 3546baa8-20ac-4bf3-9eab-e9acacdca363 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2b606 namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: cb1cccb3-dded-4ad3-b834-046e6593ed4f kind: Event lastTimestamp: "2026-07-28T16:35:37Z" message: Scaled up replica set sequence-graph-2b606-679788d849 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:37Z" name: sequence-graph-2b606.18c6815c992f279f namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 48f0263e-d801-46cf-be71-121cbea55782 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2b606 namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: e1d2e407-886f-4d40-b5d8-4b35151ceadc kind: Event lastTimestamp: "2026-07-28T16:35:37Z" message: InferenceGraph [sequence-graph-2b606] is Ready metadata: creationTimestamp: "2026-07-28T16:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:35:37Z" name: sequence-graph-2b606.18c6815c9e44463a namespace: kserve-ci-e2e-test resourceVersion: "17283" uid: ef13114f-837d-48fe-ae30-794860ab8604 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:35:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2b606 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 5fff856d-622a-4a76-81d8-89e261e26254 kind: Event lastTimestamp: "2026-07-28T16:36:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:36:22Z" name: sequence-graph-2b606.18c68160179f5f6f namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: 4a35b54b-d077-41c8-8ffe-8d850f4a05af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:35:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2b606 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 5fff856d-622a-4a76-81d8-89e261e26254 kind: Event lastTimestamp: "2026-07-28T16:36:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:36:22Z" name: sequence-graph-2b606.18c68160180c6e9b namespace: kserve-ci-e2e-test resourceVersion: "17620" uid: c19120c0-f6f2-49e2-98d7-4c6661975e46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:44:51.828083Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-92dc3-8498bd67cd-n7qfz to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:44:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:44:51Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681ddc7290f6f namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 659d409d-fbf7-422c-9bed-5c021c292ff3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-92dc3-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:52Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681ddd49eb6b5 namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: 4dfa5ab6-9971-4669-8b24-8c09de731373 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:44:52Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681de04102abf namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: 44438101-efb3-4c32-96fb-7fc1bc2b3c51 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92dc3} kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:52Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681de05429f09 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: 00b6cfda-a578-420d-be41-3dcdfdd8e647 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92dc3} kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: 'Created container: sequence-graph-92dc3' metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:52Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681de07f344af namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: f42a5f72-813b-47fd-a0d1-ae9ed6071d20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92dc3} kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:44:52Z" message: Started container sequence-graph-92dc3 metadata: creationTimestamp: "2026-07-28T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:52Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681de08a643d5 namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 762f9535-453c-49c1-91a1-a8fa7a1d98d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92dc3} kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:45:01Z" message: Stopping container sequence-graph-92dc3 metadata: creationTimestamp: "2026-07-28T16:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:01Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681e02153fb16 namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 3a609fde-eb67-4091-ab38-177bebf3d2c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92dc3} kind: Pod name: sequence-graph-92dc3-8498bd67cd-n7qfz namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 46788321-15b6-4aa8-ad9e-0736b675890a kind: Event lastTimestamp: "2026-07-28T16:45:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:29Z" name: sequence-graph-92dc3-8498bd67cd-n7qfz.18c681e0cd486235 namespace: kserve-ci-e2e-test resourceVersion: "22253" uid: 6994bc1d-00a0-4d49-8ccf-ed72741f9ebc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-92dc3-8498bd67cd namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 0c3152e5-95ac-40d1-8675-aa4d109ae732 kind: Event lastTimestamp: "2026-07-28T16:44:51Z" message: 'Created pod: sequence-graph-92dc3-8498bd67cd-n7qfz' metadata: creationTimestamp: "2026-07-28T16:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:51Z" name: sequence-graph-92dc3-8498bd67cd.18c681ddc6c0a7fa namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b1844899-2103-43af-af83-07fdb7b5740f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 02e983bc-7e17-4730-9241-0c67cfabc81f kind: Event lastTimestamp: "2026-07-28T16:44:51Z" message: Scaled up replica set sequence-graph-92dc3-8498bd67cd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:51Z" name: sequence-graph-92dc3.18c681ddc5f72aa3 namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 01906a85-63c6-4b3f-84bb-9507043fd3bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 87044929-385f-4e8e-babf-df9942b86c17 kind: Event lastTimestamp: "2026-07-28T16:44:51Z" message: InferenceGraph [sequence-graph-92dc3] is Ready metadata: creationTimestamp: "2026-07-28T16:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:51Z" name: sequence-graph-92dc3.18c681ddcb0e10de namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: 1068ef5c-f2f6-4ac1-a894-a41c40800f09 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:54:17.068986Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-2717b-5fd8b855cc-sldff to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c6826162253dfb namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 36473bd7-53fd-44a2-a68b-791a042a6fa1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c682617b513fe5 namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: 1a5c99b7-7871-42a3-a83b-2ea664411cfa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2717b} kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c682617c742b9a namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: 795e5358-f83c-46b4-8611-f6733a0289a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2717b} kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'Created container: splitter-graph-2717b' metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c682617f54a86f namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: 56559232-a8ad-49e5-bfa3-41ee925e0b9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2717b} kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Started container splitter-graph-2717b metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c682617fc679a9 namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: e6db683a-430e-4532-81c0-d01c396fa79a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2717b} kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Stopping container splitter-graph-2717b metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c68263b9bda41d namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 79fe86fd-47bf-45d7-be06-b478696ccf81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2717b} kind: Pod name: splitter-graph-2717b-5fd8b855cc-sldff namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: be4cf707-e78a-4ca5-ab2c-09f0388d2f22 kind: Event lastTimestamp: "2026-07-28T16:54:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:53Z" name: splitter-graph-2717b-5fd8b855cc-sldff.18c682641e20a070 namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: 1e9c0f61-ce30-4f7f-89ed-6e00d052177c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-2717b-5fd8b855cc namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 2f28b308-558f-44e2-9021-9d8160f02078 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'Created pod: splitter-graph-2717b-5fd8b855cc-sldff' metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b-5fd8b855cc.18c6826161852f44 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 15edced7-c966-4c73-b69b-59e2057063d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-2717b namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 687c97aa-73a7-4061-ab73-a6c86aafe8e0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Scaled up replica set splitter-graph-2717b-5fd8b855cc from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b.18c6826160b87c84 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 298370a2-00d1-4b63-920a-af33533b6535 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-2717b namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: f5e45f97-ffb6-4464-8a92-7c72914f250c kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: InferenceGraph [splitter-graph-2717b] is Ready metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:17Z" name: splitter-graph-2717b.18c6826165f401e0 namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: 85d6e13f-7178-4871-ae7a-b8804d98da15 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:55:27.339468Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27413" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b6e34-54964b94f7-q8lxq to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:55:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:55:27Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c68271be9750a3 namespace: kserve-ci-e2e-test resourceVersion: "27419" uid: cfcc223b-e429-4643-a16a-27dbdb08fc44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:27Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T16:55:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-b6e34-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:27Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c68271c75e3994 namespace: kserve-ci-e2e-test resourceVersion: "27432" uid: f1ad35f6-f1c9-41ef-971d-47a2741131bc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:28Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27424" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T16:55:28Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:55:28Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c68271fb54ef5b namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 72d4c768-b9eb-4ad2-91f1-6e9b24b49ac7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b6e34} kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T16:55:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:28Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c68271fc958688 namespace: kserve-ci-e2e-test resourceVersion: "27442" uid: ed3d0d5b-6d3a-4657-9fd1-dcc0cece2576 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b6e34} kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T16:55:28Z" message: 'Created container: splitter-graph-b6e34' metadata: creationTimestamp: "2026-07-28T16:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:28Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c68271ffd6efea namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: c65b0caf-84e7-40f3-bf20-e73dc4fff81c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b6e34} kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T16:55:28Z" message: Started container splitter-graph-b6e34 metadata: creationTimestamp: "2026-07-28T16:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:28Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c682720047477c namespace: kserve-ci-e2e-test resourceVersion: "27444" uid: 3598761e-0d1e-4cd8-96a8-df2bb58967a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b6e34} kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T17:03:41Z" message: Stopping container splitter-graph-b6e34 metadata: creationTimestamp: "2026-07-28T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:41Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c682e4e89679c3 namespace: kserve-ci-e2e-test resourceVersion: "30883" uid: 99081ada-67c7-48f4-aaba-aa3a5f5c951a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b6e34} kind: Pod name: splitter-graph-b6e34-54964b94f7-q8lxq namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: bd61f92a-f1d3-444a-b990-747317d94b74 kind: Event lastTimestamp: "2026-07-28T17:04:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:04:10Z" name: splitter-graph-b6e34-54964b94f7-q8lxq.18c682e5a2a2b2aa namespace: kserve-ci-e2e-test resourceVersion: "31124" uid: 2b87ac71-a456-4336-b907-3a1bc2458181 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b6e34-54964b94f7 namespace: kserve-ci-e2e-test resourceVersion: "27409" uid: ae2f6e72-174e-4e43-9d86-e23f91e312eb kind: Event lastTimestamp: "2026-07-28T16:55:27Z" message: 'Created pod: splitter-graph-b6e34-54964b94f7-q8lxq' metadata: creationTimestamp: "2026-07-28T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:27Z" name: splitter-graph-b6e34-54964b94f7.18c68271be34fd9a namespace: kserve-ci-e2e-test resourceVersion: "27415" uid: bd58bf4b-f7c5-4b2f-be90-c1bc9686d8ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "27407" uid: 1e55d59f-0605-4e73-94af-4f5fbd4d0ae5 kind: Event lastTimestamp: "2026-07-28T16:55:27Z" message: Scaled up replica set splitter-graph-b6e34-54964b94f7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:27Z" name: splitter-graph-b6e34.18c68271bd9a98a2 namespace: kserve-ci-e2e-test resourceVersion: "27411" uid: 4cb2ef6b-21f9-4ff7-918f-2e13e435aca3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "27426" uid: 0b424d01-8991-46d8-b895-14404263c9cd kind: Event lastTimestamp: "2026-07-28T16:55:27Z" message: InferenceGraph [splitter-graph-b6e34] is Ready metadata: creationTimestamp: "2026-07-28T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:27Z" name: splitter-graph-b6e34.18c68271c2d1dca1 namespace: kserve-ci-e2e-test resourceVersion: "27428" uid: e13b702a-0d75-4e37-8131-8a5743b69edf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: 3b38b195-8875-44e8-842c-5a19327ae5f8 kind: Event lastTimestamp: "2026-07-28T16:55:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:57Z" name: splitter-graph-b6e34.18c682753c1b5285 namespace: kserve-ci-e2e-test resourceVersion: "27670" uid: 72754f1d-a008-4f41-b532-7f5596ff3219 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: 3b38b195-8875-44e8-842c-5a19327ae5f8 kind: Event lastTimestamp: "2026-07-28T16:55:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:57Z" name: splitter-graph-b6e34.18c682753c6610b1 namespace: kserve-ci-e2e-test resourceVersion: "27671" uid: 1d0f87e4-70dc-4ced-adbf-4874d44be5c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:45:02.072753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-12f5e-predictor-597f7c7d45-5tthj to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e029ca5c4d namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: 2ffa3c02-4f29-42fb-a41d-f9e57164cc46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-12f5e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e03a851a42 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 36fc9122-5466-4eb0-bb73-bc6a5086a980 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e0674c36e6 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: 95de80cf-6673-4ee7-a638-65862df5a223 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e068b9ac3c namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: 1ec13f07-cf8a-4209-a69e-0217ad67ff85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e06cee0224 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: f28e345a-b102-40e1-80ef-71041488df17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e06d6acb6d namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: a8c99f33-3b36-4307-9fc1-c8b0b1efd3e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e06d75768e namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: a700f00a-b026-4943-b11a-9b36fc395f63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e070cf1f11 namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 8aa9ae49-9265-4882-9f41-65ab1b686888 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:03Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e0714b70bd namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: e1c1dd0d-f60a-4bf9-9077-3d52efe07921 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:45:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:40Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c681e0c7cd820d namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: ab55e725-21ad-4d6b-b7e7-d35ceaa93d8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c68261526af0ad namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: 98be31d1-916c-456a-a15f-cd56eb2d7570 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: dc1a122b-f788-4e7c-b66f-db2ee469681b kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj.18c68261526cdb44 namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 635689b7-ad54-4eb4-8f2c-91cc6bb89761 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-12f5e-predictor-597f7c7d45 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: 6276f5b2-7d4a-47b9-8e6d-3a785035bc19 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'Created pod: success-200-isvc-12f5e-predictor-597f7c7d45-5tthj' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-12f5e-predictor-597f7c7d45.18c681e029561580 namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: 4fa9c51c-9800-4d94-ad54-82382bc60a03 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: 1885ae9b-ff00-4d8e-a1df-92749ec4236c kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Scaled up replica set success-200-isvc-12f5e-predictor-597f7c7d45 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-12f5e-predictor.18c681e027541bfb namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: b6fb0f9c-29bf-445a-888c-472cafc7ba75 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: 38827bfe-69f2-42f8-807d-f5a80cfb90f5 kind: Event lastTimestamp: "2026-07-28T16:45:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:32Z" name: success-200-isvc-12f5e-predictor.18c681e3a5fbb378 namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: a750e727-f896-452c-a8b7-2fdb331cfa1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: 38827bfe-69f2-42f8-807d-f5a80cfb90f5 kind: Event lastTimestamp: "2026-07-28T16:45:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:32Z" name: success-200-isvc-12f5e-predictor.18c681e3a6ad5624 namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: d5c4a94d-09ba-4ba0-b1fc-ec5e7b2de9d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:45:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: 38827bfe-69f2-42f8-807d-f5a80cfb90f5 kind: Event lastTimestamp: "2026-07-28T16:46:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:32Z" name: success-200-isvc-12f5e-predictor.18c681eaa67e2e00 namespace: kserve-ci-e2e-test resourceVersion: "22771" uid: ada21d0e-fccf-4078-8f86-8df18a0f9422 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:45:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12f5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: 38827bfe-69f2-42f8-807d-f5a80cfb90f5 kind: Event lastTimestamp: "2026-07-28T16:46:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:46:32Z" name: success-200-isvc-12f5e-predictor.18c681eaa7663e1d namespace: kserve-ci-e2e-test resourceVersion: "22772" uid: 36830645-e04d-4a72-868b-d8eb4ccf792a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 66e1b085-fec6-4a0d-8b98-1a09984ae373 kind: Event lastTimestamp: "2026-07-28T16:45:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:50Z" name: success-200-isvc-12f5e.18c681e027679035 namespace: kserve-ci-e2e-test resourceVersion: "22425" uid: 016c81b9-1c4e-46b5-a6b8-8c750039e0e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 66e1b085-fec6-4a0d-8b98-1a09984ae373 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-12f5e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-12f5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-12f5e.18c681e02850603e namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: a4c02065-9c9b-4e5b-8838-8158ecc95a62 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 66e1b085-fec6-4a0d-8b98-1a09984ae373 kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-12f5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-12f5e.18c681e02850f8fc namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: a2bf5288-18ad-42ce-9e7a-8bc66623d0ec reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12f5e namespace: kserve-ci-e2e-test resourceVersion: "22422" uid: 66e1b085-fec6-4a0d-8b98-1a09984ae373 kind: Event lastTimestamp: "2026-07-28T16:45:50Z" message: InferenceService [success-200-isvc-12f5e] is Ready metadata: creationTimestamp: "2026-07-28T16:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:45:50Z" name: success-200-isvc-12f5e.18c681eb847d381e namespace: kserve-ci-e2e-test resourceVersion: "22424" uid: 89915077-15cc-41da-845f-5e4b74589a46 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:53:17.014220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2717b-predictor-5c68f8885c-sjxmd to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c682536699c744 namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 06fb03fa-0942-4d80-8e63-09539ca0ebac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25901" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c682538056b431 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 6088525e-99b3-46d4-91c9-0e1e8f1bd6e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c682538191b2aa namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: f3de8314-eae8-4567-a98a-c5cb754604b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68253849028b7 namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: fed21fed-720b-4bd5-a236-3a857c6cc8fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68253858259fe namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: eec39ada-41d4-492a-a818-320d7070cf29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68253858c8e41 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: 54aad072-2561-47d2-a9ed-4c2d4e0f67eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c6825388188016 namespace: kserve-ci-e2e-test resourceVersion: "25950" uid: 267b6f99-8323-432f-a038-dfc2ef99a6d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c6825388c5ede6 namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: bec0fedd-44b7-4080-90dd-4fa5b64b3160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:53:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:54Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68253d9201543 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 96a0e8bc-7325-466d-b101-c120bb2b40fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68263c05f714f namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: d12b3fd0-92a0-4314-9b4d-e83bc734a337 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68263c05ffb83 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 20d50f22-55bf-4cfe-9e16-f85bdb7e6327 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 7ec846e8-0d27-4b69-bfe0-538d5f6325de kind: Event lastTimestamp: "2026-07-28T16:54:29Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:29Z" name: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd.18c68264615598d6 namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: 02d5ff8c-5292-4c60-87f1-338dea5ab247 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2717b-predictor-5c68f8885c namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: ddf5c55c-b03b-4cde-aa4f-aa66313c743f kind: Event lastTimestamp: "2026-07-28T16:53:17Z" message: 'Created pod: success-200-isvc-2717b-predictor-5c68f8885c-sjxmd' metadata: creationTimestamp: "2026-07-28T16:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:17Z" name: success-200-isvc-2717b-predictor-5c68f8885c.18c68253662cfd60 namespace: kserve-ci-e2e-test resourceVersion: "25879" uid: 413ab913-2bf1-41b2-8f4f-471b14f9f71e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25859" uid: 04a098dd-8f3f-47a0-89c7-08dd6e6935c4 kind: Event lastTimestamp: "2026-07-28T16:53:16Z" message: Scaled up replica set success-200-isvc-2717b-predictor-5c68f8885c from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:16Z" name: success-200-isvc-2717b-predictor.18c682536306db10 namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: f2b22dd6-65d4-42d5-8f65-bbcfee8dc89d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: e34185e8-5a4b-4b42-9557-3a7e957ee6d1 kind: Event lastTimestamp: "2026-07-28T16:53:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:31Z" name: success-200-isvc-2717b-predictor.18c68256e2519cfa namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: 0aa2c163-4c96-4f25-8976-e86dd3bcc39a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: e34185e8-5a4b-4b42-9557-3a7e957ee6d1 kind: Event lastTimestamp: "2026-07-28T16:53:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:53:31Z" name: success-200-isvc-2717b-predictor.18c68256e2bdd376 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 11be70bb-7c9d-4264-b8cd-3b4d3be76e54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:53:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: e34185e8-5a4b-4b42-9557-3a7e957ee6d1 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-2717b-predictor.18c6825a61a20016 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: 21a3e6fa-3232-4ac7-a2b2-d4cced3677f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:53:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2717b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: e34185e8-5a4b-4b42-9557-3a7e957ee6d1 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-2717b-predictor.18c6825a620b8b29 namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 62b29065-55b2-4f19-a272-cd0c28656c79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: 8111e153-485f-45bc-b32d-f5b0be5d0374 kind: Event lastTimestamp: "2026-07-28T16:54:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:05Z" name: success-200-isvc-2717b.18c68253633637e2 namespace: kserve-ci-e2e-test resourceVersion: "26350" uid: 9a580fca-817a-438b-9dcc-b3f6da7e348a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: 8111e153-485f-45bc-b32d-f5b0be5d0374 kind: Event lastTimestamp: "2026-07-28T16:53:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-2717b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2717b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:53:16Z" name: success-200-isvc-2717b.18c68253639fb9ed namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: 7f64073a-c639-4741-8592-db82d575d4d8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: 8111e153-485f-45bc-b32d-f5b0be5d0374 kind: Event lastTimestamp: "2026-07-28T16:53:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2717b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:53:16Z" name: success-200-isvc-2717b.18c6825363a00416 namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: 3155f2c6-fa9b-430c-aa85-5b2d790edf43 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2717b namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 8111e153-485f-45bc-b32d-f5b0be5d0374 kind: Event lastTimestamp: "2026-07-28T16:54:05Z" message: InferenceService [success-200-isvc-2717b] is Ready metadata: creationTimestamp: "2026-07-28T16:54:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:05Z" name: success-200-isvc-2717b.18c6825e93e940a3 namespace: kserve-ci-e2e-test resourceVersion: "26349" uid: e624a897-29d8-40fe-8b27-36064a5d8a16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:34:37.025256Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2b606-predictor-64d46d8575-sbrbh to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:34:37Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814ea20e649f namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: cded4b73-7b09-40d6-b5c1-39c21bbaf4a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2b606-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814eab93f4a2 namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: 9ba5c8bc-e9be-45c7-b773-e697a08dcf2e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:34:38Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f15b309c0 namespace: kserve-ci-e2e-test resourceVersion: "16757" uid: 5250e5cc-973d-4873-b025-e97bffbb389a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f17411f2b namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: 162e61a7-b645-4235-b89e-efe16d370924 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:39Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f1a138584 namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: dff10e70-6af8-4550-adb5-92018ee5cb73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:39Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f1a7c5874 namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 81631e55-a0ec-4d5a-a93c-e28bb96a1009 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:39Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f1a86a143 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: c2f02018-3054-4d95-8785-03c0a75723d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:39Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f1d3f0b13 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 24f7b3fc-b8ab-47b7-86a9-08f1c9157631 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:34:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:39Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f1da0619d namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: c058313d-4e85-472b-81ea-f9c26704610b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:35:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:35:15Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c6814f4ad49e69 namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: 7b1b927c-cf6a-4818-aef1-cc6881282c83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c681cfcc0ce447 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: 661f9dfd-a234-4c3a-b9c2-ce08edb0914c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: dd8f78ab-362e-4ef5-8aa1-4362e5de1374 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-2b606-predictor-64d46d8575-sbrbh.18c681cfcc0d5fff namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: ad4ac508-42f0-4a78-841d-b28dd38de84e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2b606-predictor-64d46d8575 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 0cc34ef0-8061-4162-b4b5-1afcd2ad59a4 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'Created pod: success-200-isvc-2b606-predictor-64d46d8575-sbrbh' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:37Z" name: success-200-isvc-2b606-predictor-64d46d8575.18c6814ea196ead7 namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 1a4a11c3-0cc3-4a5b-9ddd-13da9039ded3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: 333c68e3-2d0d-44a7-b9e7-dc44f222ddd8 kind: Event lastTimestamp: "2026-07-28T16:34:36Z" message: Scaled up replica set success-200-isvc-2b606-predictor-64d46d8575 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:36Z" name: success-200-isvc-2b606-predictor.18c6814e9fb34cae namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: a65aaea4-1637-432e-adbc-701579caadbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:34:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 7294d9c6-ba96-42fa-be0c-7aad118c59df kind: Event lastTimestamp: "2026-07-28T16:35:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:22Z" name: success-200-isvc-2b606-predictor.18c681521ec197ce namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 4c30c7f8-d236-4177-be9f-930ac229bbed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:34:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 7294d9c6-ba96-42fa-be0c-7aad118c59df kind: Event lastTimestamp: "2026-07-28T16:35:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:22Z" name: success-200-isvc-2b606-predictor.18c681521f2661fd namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: 4dc17577-16ef-44ae-a1be-e0f4ce5ef5ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 7294d9c6-ba96-42fa-be0c-7aad118c59df kind: Event lastTimestamp: "2026-07-28T16:36:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:36:07Z" name: success-200-isvc-2b606-predictor.18c681559e8200b1 namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: b4f56c82-b857-4b38-8b25-0db0d5fbe812 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2b606-predictor namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 7294d9c6-ba96-42fa-be0c-7aad118c59df kind: Event lastTimestamp: "2026-07-28T16:36:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:36:07Z" name: success-200-isvc-2b606-predictor.18c681559ee61ce3 namespace: kserve-ci-e2e-test resourceVersion: "17517" uid: 54895aab-c1c1-4684-a01d-6853a7866c7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:34:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2b606 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 322a0e51-7a6e-480b-9dde-60c4f8e081c5 kind: Event lastTimestamp: "2026-07-28T16:35:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:35:25Z" name: success-200-isvc-2b606.18c6814ea04abcc6 namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: 6844d6ea-afa2-4f33-9131-df8a6f1b3ef1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2b606 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: 322a0e51-7a6e-480b-9dde-60c4f8e081c5 kind: Event lastTimestamp: "2026-07-28T16:35:25Z" message: InferenceService [success-200-isvc-2b606] is Ready metadata: creationTimestamp: "2026-07-28T16:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:35:25Z" name: success-200-isvc-2b606.18c6815a059e2100 namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 8b582200-5417-40b6-9813-915e079a94fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:42:51.861613Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:42:51Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1d899f89f namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 1cacd06b-dbbb-4d45-bcc9-9e4b2f2e726b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1f2cad379 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: acd38c52-b309-4aa9-98d5-53525b26ba87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1f410d125 namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: cb0ef4ed-d401-43f5-af52-18ce28e30723 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1f76637c0 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 0c870b39-a45b-4859-9577-93ef82fa6050 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1f7dbedea namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: b9d19f72-1e18-4149-b795-ac184794b2ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1f7e6be33 namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 8cd1e175-6759-464b-8a87-6785d495c69d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1fa94614c namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: f0d9ece3-e13f-41df-b4f4-424904a592f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c1fb09fb51 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: a95441e1-8e79-4422-8849-e9556c88bf26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:43:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:30Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681c25ecb1404 namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 73a1e6d2-53a5-4276-8c75-27939d7c9064 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681d2321acb93 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 7d33fec9-cfc0-4f49-9b71-a8252acab5d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681d2321b5c96 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 18a6e326-70be-4287-aaa9-a1d657db94b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 4e8697a3-088c-43ab-944d-223d98a45b03 kind: Event lastTimestamp: "2026-07-28T16:44:05Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:05Z" name: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m.18c681d2e6f072ba namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: 52fd648a-1768-4e31-81aa-90ff100955fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-68c6a-predictor-78785fb6b7 namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: e8b34556-fa76-4877-adc9-582df87c0544 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: 'Created pod: success-200-isvc-68c6a-predictor-78785fb6b7-4wt4m' metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:42:51Z" name: success-200-isvc-68c6a-predictor-78785fb6b7.18c681c1d8337fe3 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 3340d48c-9f41-4fea-b76e-f1ce64a690c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: c4aaa021-d3d2-4c33-b6cd-cb73e52d61a8 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: Scaled up replica set success-200-isvc-68c6a-predictor-78785fb6b7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:42:51Z" name: success-200-isvc-68c6a-predictor.18c681c1d515f5b6 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 0dbe326b-6ad8-4e69-be66-4a6fa0443766 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 5f37d347-c45b-4d62-a36b-1307088e8085 kind: Event lastTimestamp: "2026-07-28T16:43:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:06Z" name: success-200-isvc-68c6a-predictor.18c681c5544197b1 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: bf8e051f-9ae1-4611-b8a9-c558dedbef4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 5f37d347-c45b-4d62-a36b-1307088e8085 kind: Event lastTimestamp: "2026-07-28T16:43:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:06Z" name: success-200-isvc-68c6a-predictor.18c681c554a622be namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 7554ec0a-815b-4291-acbe-7a0c3c2d3ee3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:43:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 5f37d347-c45b-4d62-a36b-1307088e8085 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-68c6a-predictor.18c681c8d3a5c6fe namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: eb005bb2-0195-4d24-8a49-7aa7e34296c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:43:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 5f37d347-c45b-4d62-a36b-1307088e8085 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-68c6a-predictor.18c681c8d40b3d8e namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 840f07de-ab38-4988-bf19-d7317db1e005 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 0b288c2f-b1d5-4eb2-8e9a-96c0e11b2d7b kind: Event lastTimestamp: "2026-07-28T16:43:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:40Z" name: success-200-isvc-68c6a.18c681c1d53c1600 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 8161c997-953c-4854-b9d9-5e17a839d1ea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 0b288c2f-b1d5-4eb2-8e9a-96c0e11b2d7b kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-68c6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-68c6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:42:51Z" name: success-200-isvc-68c6a.18c681c1d5b5d277 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 05cefee4-6622-4fe3-a355-c812d9f45d0a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 0b288c2f-b1d5-4eb2-8e9a-96c0e11b2d7b kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-68c6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:42:51Z" name: success-200-isvc-68c6a.18c681c1d5b614bc namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 0f54956f-29a8-487e-a872-a4e741615074 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68c6a namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 0b288c2f-b1d5-4eb2-8e9a-96c0e11b2d7b kind: Event lastTimestamp: "2026-07-28T16:43:40Z" message: InferenceService [success-200-isvc-68c6a] is Ready metadata: creationTimestamp: "2026-07-28T16:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:40Z" name: success-200-isvc-68c6a.18c681cd199d0b2b namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 6665cd36-f402-4af5-90e1-30902ff39ae5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:33:36.966248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140a642b645 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: b05f1bd5-e99a-46b9-9de1-ca18dc11ae27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7ba9d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:37Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140b298c897 namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 727c322f-f5f7-4751-b18a-931c3ee03d1a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:37Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:33:37Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140e3b05b00 namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: b59e1612-e5a5-48e3-bfc0-2f105b5682b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140e4e68df5 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: a7a69b8a-8e94-43dd-bfea-4ca616faa87b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140e857952d namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: c109aee0-3d00-4a94-b456-57d2dd8610e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140e8c4e348 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: a3beaa42-a4e3-4865-8a48-b3503767e155 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140e8cf08fd namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: 67781f55-dfb5-429b-9877-af61c93d63de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140ebcb33dd namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: ad7ae44f-e141-49b9-841e-f642e83c98b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c68140ec3af339 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: f0c13c11-8e31-4754-85f9-3301d1256b00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:42:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:54Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c6814105f6fd2e namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: cc9ac548-e1df-4cd9-9f5b-e8c6d44d7d55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c681c1e6a489da namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: e918898a-21f4-403e-8def-17e6ec672e3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:42:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:52Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c681c1e6a4fc8a namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: a804e6a6-b8d7-4529-9773-32384cffb982 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 34ba2157-d9a4-42c8-834c-f349e8df2b76 kind: Event lastTimestamp: "2026-07-28T16:42:54Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:54Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq.18c681c27a67b000 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 1f3358c0-0ef3-4104-b524-d12493e17712 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7ba9d-predictor-68cbcf59bf namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 3aeeca0e-9b2d-4f0e-9f24-70170f52df5f kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: 'Created pod: success-200-isvc-7ba9d-predictor-68cbcf59bf-jt9kq' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-7ba9d-predictor-68cbcf59bf.18c68140a5d95971 namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: 332ed6e9-c9f1-4a40-8e1a-0c5bf8791a1c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 202d9303-3633-44f8-9a43-7869726a9eba kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: Scaled up replica set success-200-isvc-7ba9d-predictor-68cbcf59bf from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-7ba9d-predictor.18c68140a3d902eb namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 06ae342d-7edc-44b5-af16-6a3b68cc5de7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 44b8ac68-f7df-42ed-bde4-629ebac23c85 kind: Event lastTimestamp: "2026-07-28T16:33:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:51Z" name: success-200-isvc-7ba9d-predictor.18c68144223133e5 namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: e41b1da4-d806-475a-929f-b7671b60b51c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 44b8ac68-f7df-42ed-bde4-629ebac23c85 kind: Event lastTimestamp: "2026-07-28T16:33:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:51Z" name: success-200-isvc-7ba9d-predictor.18c681442297ac02 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: a11acb88-5ae2-4610-8700-c5385e4f6140 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:34:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 44b8ac68-f7df-42ed-bde4-629ebac23c85 kind: Event lastTimestamp: "2026-07-28T16:34:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:52Z" name: success-200-isvc-7ba9d-predictor.18c68147a1d00423 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 282e11fa-9faf-469b-a409-c09108680cc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:34:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ba9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 44b8ac68-f7df-42ed-bde4-629ebac23c85 kind: Event lastTimestamp: "2026-07-28T16:34:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:52Z" name: success-200-isvc-7ba9d-predictor.18c68147a243273c namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: ca392426-9cd5-49bc-9416-56e77dd18922 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 37983729-bb7b-41af-9679-d45035a2e37c kind: Event lastTimestamp: "2026-07-28T16:34:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:24Z" name: success-200-isvc-7ba9d.18c68140a3d1e6c0 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: b5a17c87-36f2-4813-8df2-50334ab80803 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 37983729-bb7b-41af-9679-d45035a2e37c kind: Event lastTimestamp: "2026-07-28T16:34:24Z" message: InferenceService [success-200-isvc-7ba9d] is Ready metadata: creationTimestamp: "2026-07-28T16:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:24Z" name: success-200-isvc-7ba9d.18c6814bc0676e23 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 4bb38421-40c4-47c4-8715-6bb192148958 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:43:51.837625Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-92dc3-predictor-7d6df59697-gbssk to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cfcf735779 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: db2d7263-a26b-4175-99f3-588ece3ba9b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cfe9478b76 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 8384ff20-3451-4915-b337-7d98d9d5aa13 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cfeaaa5e5d namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: b53804ae-f12c-49d6-a1cc-963868adfecc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cfed7398ce namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: a50b3570-321c-440d-9b5f-c6797b5a54aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cfeded1759 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: b933c80e-2708-4188-94d5-7990d7d54dfd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cfedf69b0c namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: bd7ffbd8-df71-4cba-8d74-51c478f9a9ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cff2c98eeb namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: a98529fc-1e8c-41ba-b3fd-192e390a288e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:43:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:52Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681cff341e509 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: e60082ef-d5ab-48a6-ab38-1d487e815317 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:44:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:30Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681d067c35606 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: bf375c87-f396-4e79-9b7a-ea1449e37db2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681e027f5d615 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: c35ce6d2-bae4-4fd6-b9be-9dc1ea742874 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c2052a1a-c107-4c2c-965a-232dc354050d kind: Event lastTimestamp: "2026-07-28T16:45:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:45:02Z" name: success-200-isvc-92dc3-predictor-7d6df59697-gbssk.18c681e027f6fa1f namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: 0736f588-fc94-42a6-8833-0a386d0bc5e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-92dc3-predictor-7d6df59697 namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: f4919c83-369c-4635-a47f-6a3a7e4fca10 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'Created pod: success-200-isvc-92dc3-predictor-7d6df59697-gbssk' metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-92dc3-predictor-7d6df59697.18c681cfcef660a2 namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: f88b542c-9bc1-4429-83a5-2e8543aef57f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: b296248a-3ab0-4583-941e-11746b3497e8 kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: Scaled up replica set success-200-isvc-92dc3-predictor-7d6df59697 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-92dc3-predictor.18c681cfcc9373bd namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: 7fd5b7a4-7b61-4013-a553-8cebf97eba82 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 7d3195d3-37e6-4d1f-acd5-ee87599eb087 kind: Event lastTimestamp: "2026-07-28T16:44:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:36Z" name: success-200-isvc-92dc3-predictor.18c681d34b467b37 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 616324c0-7226-4fbc-ac9d-f9007a1a48b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 7d3195d3-37e6-4d1f-acd5-ee87599eb087 kind: Event lastTimestamp: "2026-07-28T16:44:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:36Z" name: success-200-isvc-92dc3-predictor.18c681d34bb87e23 namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: 1c3a8f99-f7eb-4a5c-8306-0b09416090dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 7d3195d3-37e6-4d1f-acd5-ee87599eb087 kind: Event lastTimestamp: "2026-07-28T16:44:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:51Z" name: success-200-isvc-92dc3-predictor.18c681d6ca9a5770 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: 40988c4a-aba5-4c74-af30-9b2e0818b7eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92dc3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 7d3195d3-37e6-4d1f-acd5-ee87599eb087 kind: Event lastTimestamp: "2026-07-28T16:44:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:51Z" name: success-200-isvc-92dc3-predictor.18c681d6cb583aa6 namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: b2b9aab8-f96c-4788-8884-efb36d9ae243 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 342f8c6a-45bc-4762-b61d-2009e07ccb7a kind: Event lastTimestamp: "2026-07-28T16:44:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:40Z" name: success-200-isvc-92dc3.18c681cfcc95b8a1 namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: fcedef14-cb24-4347-881b-e4f1653ba75f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 342f8c6a-45bc-4762-b61d-2009e07ccb7a kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-92dc3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-92dc3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-92dc3.18c681cfcd2aa5da namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: a7ce1b69-7943-4082-abf8-27f17b7391ac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:43:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 342f8c6a-45bc-4762-b61d-2009e07ccb7a kind: Event lastTimestamp: "2026-07-28T16:43:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-92dc3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:43:51Z" name: success-200-isvc-92dc3.18c681cfcd2b0922 namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 9dd83857-6c2c-41a9-9909-31f9e98acd6d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92dc3 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 342f8c6a-45bc-4762-b61d-2009e07ccb7a kind: Event lastTimestamp: "2026-07-28T16:44:40Z" message: InferenceService [success-200-isvc-92dc3] is Ready metadata: creationTimestamp: "2026-07-28T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:40Z" name: success-200-isvc-92dc3.18c681db2308b1ea namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: ebc035a0-78ba-4869-abc7-54dc313e9497 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:06.682327Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:06Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812ba0eb509e namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: e15dd832-12c9-45ec-8b25-61ca713a0d4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b2ad5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:06Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812baeae0404 namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: f52ecdc5-7631-4088-ad4d-4417bf098a6e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:07Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812bde85a69e namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: f84846f6-daf0-491c-8b28-207f0513097b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:07Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T16:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:07Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812bdfc77ba6 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 71612481-ffd0-41fc-b68a-d072ec17aa97 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:19Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" in 12.045s (12.045s including waiting). Image size: 1334930032 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:19Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812eadba54b3 namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: eda35431-3359-4744-bafd-1abd0ee6f6c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:19Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812eb0d43392 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 67ea4652-9228-49f8-acaf-26e97334f020 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:19Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812eb149614c namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 15fd685b-1751-47df-80a7-958f36f89d5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:19Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812eb1593f8f namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 0b949839-d8dc-4630-b282-c747f660688d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.229s (2.229s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:22Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812f363ae295 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 9b72e9cf-e17d-422b-8817-64da7431446d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:22Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812f39622a24 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 7b86ec0f-a8d9-4dd8-933f-109bb80bec1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:32:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:22Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812f39d9cbed namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: b8569532-32bb-4956-99da-815f4ad53a26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c6812f401381f1 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 6fd1d400-681e-4e16-b6b3-4679053a76ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c68140a28cee8f namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: bc3a49c8-d185-4ed1-8d98-e757350149e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c68140a28d7504 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 88b582c4-03da-4886-aedf-2c224ea74390 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 225e1d35-44de-4ab7-b20e-d543efa4539b kind: Event lastTimestamp: "2026-07-28T16:33:38Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:38Z" name: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4.18c68140f23e52ac namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 2b0d3974-c39e-41ec-a3b0-c399dbf5cb45 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2ad5-predictor-666fbfd487 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 1763acfc-5e57-418c-b7ad-6c1ca9681806 kind: Event lastTimestamp: "2026-07-28T16:32:06Z" message: 'Created pod: success-200-isvc-b2ad5-predictor-666fbfd487-dt8k4' metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:06Z" name: success-200-isvc-b2ad5-predictor-666fbfd487.18c6812ba080e530 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c82ae3df-f498-4698-b5f9-a991aeae8e24 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: fd9acc93-2ae5-43ea-9987-2c394494c0d2 kind: Event lastTimestamp: "2026-07-28T16:32:06Z" message: Scaled up replica set success-200-isvc-b2ad5-predictor-666fbfd487 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:06Z" name: success-200-isvc-b2ad5-predictor.18c6812b9d2c77e1 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 62081fdf-7655-4408-956c-3288db441ff9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:32:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 3c5c4b96-9f6e-41d1-8737-9f06758aa288 kind: Event lastTimestamp: "2026-07-28T16:33:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:06Z" name: success-200-isvc-b2ad5-predictor.18c6812f227afab6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: e841f758-2df4-4677-9ca0-8991fdcbf055 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:32:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 3c5c4b96-9f6e-41d1-8737-9f06758aa288 kind: Event lastTimestamp: "2026-07-28T16:33:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:06Z" name: success-200-isvc-b2ad5-predictor.18c6812f22e97914 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 62818df0-4df1-417e-86c6-ea4f395006b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 3c5c4b96-9f6e-41d1-8737-9f06758aa288 kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-b2ad5-predictor.18c6813620c41ec8 namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: f10bd47a-704d-4354-a0fa-78addf3c80ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2ad5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 3c5c4b96-9f6e-41d1-8737-9f06758aa288 kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:36Z" name: success-200-isvc-b2ad5-predictor.18c68136212f8d13 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 22211619-29d5-4568-a71d-c88c0fbf38f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2ad5 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 21e126e7-137d-4536-9e16-05e98830c41b kind: Event lastTimestamp: "2026-07-28T16:33:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:08Z" name: success-200-isvc-b2ad5.18c6812ba317fe68 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 3ec5fc57-892c-4a36-998e-ffaafd7a4b16 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2ad5 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 21e126e7-137d-4536-9e16-05e98830c41b kind: Event lastTimestamp: "2026-07-28T16:33:08Z" message: InferenceService [success-200-isvc-b2ad5] is Ready metadata: creationTimestamp: "2026-07-28T16:33:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:08Z" name: success-200-isvc-b2ad5.18c68139fb2051e1 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: dd5dd652-17bb-42ce-828a-3098424698c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:54:27.297170Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:54:27Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c68263c3ca7378 namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: ee1c76aa-f5ce-4da7-b017-75de1d844b15 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b6e34-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:27Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c68263cd511bee namespace: kserve-ci-e2e-test resourceVersion: "26805" uid: 1e979b39-a3dd-4f95-a651-68aa1abd8960 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c68264017243f0 namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: c580afd4-62f7-45e2-8014-dcb03384c981 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c6826402eb3a43 namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 918daced-e889-4dd5-a988-815118ecbc72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c6826405f4dc9e namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: a8366472-77d2-4802-bc71-9c54a2e29a72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c6826406670888 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 26e2e498-7f5c-4694-b2d9-09183e219ed3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c68264066fe653 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: c1c97fcd-05a1-4077-8f85-8a4a34f3e419 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c68264097a6823 namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: 5ef71ddb-9318-42d4-8084-7ca776550c33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:54:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:28Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c6826409f1cf84 namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: 52710c31-78a7-4897-b3b4-867006fad055 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T16:55:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:06Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c682647528296c namespace: kserve-ci-e2e-test resourceVersion: "27193" uid: b580b16d-ab45-40f2-81bb-5d5551915159 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T17:03:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:42Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c682e4efae3b0b namespace: kserve-ci-e2e-test resourceVersion: "30907" uid: 6e3ade01-bb06-4932-9bac-ddf6a3aacd71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 309f7841-d173-466f-a580-50dda84e1758 kind: Event lastTimestamp: "2026-07-28T17:03:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:03:42Z" name: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz.18c682e4efb2f870 namespace: kserve-ci-e2e-test resourceVersion: "30910" uid: fbd3b61e-f5ff-4bf9-9b9f-91f74613855a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b6e34-predictor-64bd5d8459 namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2a1813eb-0c33-451b-a9cd-8a25b7149c3e kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: 'Created pod: success-200-isvc-b6e34-predictor-64bd5d8459-mtdzz' metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:27Z" name: success-200-isvc-b6e34-predictor-64bd5d8459.18c68263c36a699d namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: 64d75e3f-300d-4f7d-ad7b-f2631a24d606 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 0cb0e800-a177-4cdf-962d-0c9e6bcb8ceb kind: Event lastTimestamp: "2026-07-28T16:54:27Z" message: Scaled up replica set success-200-isvc-b6e34-predictor-64bd5d8459 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:27Z" name: success-200-isvc-b6e34-predictor.18c68263c199840a namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 95f51038-d8db-4e05-a7eb-ad5e84f150f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: c6e52434-6262-44dc-a0bd-426932e3b4b9 kind: Event lastTimestamp: "2026-07-28T16:54:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:57Z" name: success-200-isvc-b6e34-predictor.18c682674032c6d5 namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: 238aba42-a30a-4dc4-a5d7-181b38f33a10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: c6e52434-6262-44dc-a0bd-426932e3b4b9 kind: Event lastTimestamp: "2026-07-28T16:54:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:57Z" name: success-200-isvc-b6e34-predictor.18c68267409752c0 namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: d8c323b3-76c6-4e8c-944a-a94a53fba6d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:55:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: c6e52434-6262-44dc-a0bd-426932e3b4b9 kind: Event lastTimestamp: "2026-07-28T16:55:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:42Z" name: success-200-isvc-b6e34-predictor.18c6826e3e33e2d5 namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: faf77d0a-3bcd-48a0-ac90-dcdd8f6019e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:55:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: c6e52434-6262-44dc-a0bd-426932e3b4b9 kind: Event lastTimestamp: "2026-07-28T16:55:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:42Z" name: success-200-isvc-b6e34-predictor.18c6826e3ea0d394 namespace: kserve-ci-e2e-test resourceVersion: "27561" uid: b0b70c92-0e22-4b03-b8bf-b8d4e8c9cb14 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 0c4fc4ec-bb6e-4c38-b468-535d36ac63e3 kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:16Z" name: success-200-isvc-b6e34.18c68263c1aa7676 namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: be3cdb98-f435-480c-a0d5-ba44244bc4e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6e34 namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: 0c4fc4ec-bb6e-4c38-b468-535d36ac63e3 kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: InferenceService [success-200-isvc-b6e34] is Ready metadata: creationTimestamp: "2026-07-28T16:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:16Z" name: success-200-isvc-b6e34.18c6826f2fb0a956 namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: 2f9c73bb-7073-48ae-be77-214476482945 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:54:16.872735Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c68261567241df namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 22396ab1-af90-4451-80ab-bd1c1a171181 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c68261708b9745 namespace: kserve-ci-e2e-test resourceVersion: "26582" uid: a2a22d78-3321-4eba-b4a4-8592fa701ebb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c6826171e5639f namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: 5dfd8b2f-81d1-4ab2-9402-3e58c9f8ab91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c6826174eadbf1 namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: aa93577e-28f3-4a25-979f-a8d2e5f7d7d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c682617564bc9b namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: e7b589b5-28d9-45d7-ba22-6fd25268021d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c6826175705ea2 namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7c5988b3-ed39-424e-9ba2-bc3c5f96885b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c6826178e927d4 namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 5f2ffce8-6c5a-4a76-95da-5a30e808a5a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:17Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c682617962bdb0 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: c7d7d7f7-7409-44d4-b1e7-c9e63a664f12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T16:54:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:54:55Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c68261e1cbb143 namespace: kserve-ci-e2e-test resourceVersion: "27075" uid: a2ed0547-cd1f-42fc-8002-6aa0c3edbc07 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T17:11:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:11:36Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c683535b388e09 namespace: kserve-ci-e2e-test resourceVersion: "34187" uid: 744c6d28-10df-48d6-80df-55f4f2283a8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7e6417d3-5731-474e-8805-7c66df1f2ed0 kind: Event lastTimestamp: "2026-07-28T17:11:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:11:36Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7.18c683535b39807d namespace: kserve-ci-e2e-test resourceVersion: "34188" uid: 2035e9da-8846-4920-a31c-9d63f5167780 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b9a8c-predictor-7dd8d57945 namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: 4ae22f80-06ce-478d-b5c4-077a99fb2091 kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: 'Created pod: success-200-isvc-b9a8c-predictor-7dd8d57945-lrxn7' metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-b9a8c-predictor-7dd8d57945.18c682615603a673 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 968c849a-bb8e-4bf0-bead-9b6f709379d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 6c66fd6e-2a43-4c62-a3c3-b6844f31c330 kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: Scaled up replica set success-200-isvc-b9a8c-predictor-7dd8d57945 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-b9a8c-predictor.18c682615449d076 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 537ade58-2200-46a8-858d-353c55f061e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 6bb00376-831c-4805-be42-f3dcdc40a5bf kind: Event lastTimestamp: "2026-07-28T16:54:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:46Z" name: success-200-isvc-b9a8c-predictor.18c68264d34de1b1 namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: df30ea65-5185-4034-9109-d21a296a54e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:54:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 6bb00376-831c-4805-be42-f3dcdc40a5bf kind: Event lastTimestamp: "2026-07-28T16:54:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:54:46Z" name: success-200-isvc-b9a8c-predictor.18c68264d3b4404c namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 7e27b9f0-f799-4ed5-86b5-298d04c84586 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 6bb00376-831c-4805-be42-f3dcdc40a5bf kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:16Z" name: success-200-isvc-b9a8c-predictor.18c6826bd1ee2955 namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: dfbb1ccd-9481-478c-9f09-09a00c279527 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 6bb00376-831c-4805-be42-f3dcdc40a5bf kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:16Z" name: success-200-isvc-b9a8c-predictor.18c6826bd25823c8 namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: 72ccccc8-c84e-41fc-b18a-e72d6fa2c09f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 4ab48538-47be-4f0e-8ca4-9868a16f95a1 kind: Event lastTimestamp: "2026-07-28T16:55:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:05Z" name: success-200-isvc-b9a8c.18c682615442ed1f namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: dfb81626-362a-4680-9846-2e1cd8d9986a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 4ab48538-47be-4f0e-8ca4-9868a16f95a1 kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-b9a8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b9a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-b9a8c.18c6826154fea32a namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 25f4d993-1cd1-45d1-9f4e-1b4dbc4a3880 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:54:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 4ab48538-47be-4f0e-8ca4-9868a16f95a1 kind: Event lastTimestamp: "2026-07-28T16:54:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b9a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:54:16Z" name: success-200-isvc-b9a8c.18c6826154fef6e0 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 361f521f-d28f-4eda-bb30-2a13f8c891f0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "27172" uid: 4ab48538-47be-4f0e-8ca4-9868a16f95a1 kind: Event lastTimestamp: "2026-07-28T16:55:05Z" message: InferenceService [success-200-isvc-b9a8c] is Ready metadata: creationTimestamp: "2026-07-28T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:05Z" name: success-200-isvc-b9a8c.18c6826c9ce3eb5f namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: ef5425e0-0e14-410a-92fd-1b2f755db5a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:44:02.112140Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3541-predictor-c87f9b99-wtgfm to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d233dbf878 namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: ba93270d-a204-4cae-89f0-9e962a9f050f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d24db71684 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: bcc9e67b-0731-456b-a169-0fc7f3928eac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d24f33468a namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: d4891cd9-67b1-4a68-9d34-1897504e2a85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d253162b29 namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: a4b3d53e-9c5c-432a-92fe-63a93a6c5991 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d253900773 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 7b350a35-6c7e-4501-b52a-ee0148664bcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d2539a3255 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: cbfffbd8-6fef-40c0-8526-7744b35c0d7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d256aaa518 namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 57e6c6b8-eee8-46fc-9b01-8500775535e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d25720b135 namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: b96e3248-604f-4f54-8ee9-66ece2c549e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:53:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:19Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c681d282f52bdc namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: ae670ed6-a8e9-4ac3-a3f1-329d730082c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:53:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:16Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c68253639671a9 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: dde3bd97-d2b4-4fb9-ae3a-276a88fcdfa4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:53:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:16Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c682536396f434 namespace: kserve-ci-e2e-test resourceVersion: "25873" uid: 5f2f063f-ac36-4790-899f-bde2575cba86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:53:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 714337c0-47d4-49a8-83fe-f9861c5e6a8f kind: Event lastTimestamp: "2026-07-28T16:53:19Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:53:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:53:19Z" name: success-200-isvc-c3541-predictor-c87f9b99-wtgfm.18c68253f755202c namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: e2ee4ce6-7845-48ff-9055-4977f0dc8a89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3541-predictor-c87f9b99 namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: bad0aa4b-e38e-4d09-8d38-7f1a0b5efd9c kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Created pod: success-200-isvc-c3541-predictor-c87f9b99-wtgfm' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor-c87f9b99.18c681d2337ebcb6 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: cf9de9e2-534e-4cfc-b327-200f9d15ae55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 5bd8f523-99f6-4ea0-b09f-7e966cba50ea kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: Scaled up replica set success-200-isvc-c3541-predictor-c87f9b99 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541-predictor.18c681d231ecf9b8 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c58b315c-5322-440a-917c-eb7515314554 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 6e8c0c05-a3c2-436d-baca-9dcca318404b kind: Event lastTimestamp: "2026-07-28T16:44:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:32Z" name: success-200-isvc-c3541-predictor.18c681d5b07aa0e6 namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: a00cd894-1532-4cfd-9513-d63a2cb3d287 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:44:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 6e8c0c05-a3c2-436d-baca-9dcca318404b kind: Event lastTimestamp: "2026-07-28T16:44:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:44:32Z" name: success-200-isvc-c3541-predictor.18c681d5b0e77da3 namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 78e0a94d-0dc7-4a49-bf0f-80441b82ce50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 6e8c0c05-a3c2-436d-baca-9dcca318404b kind: Event lastTimestamp: "2026-07-28T16:45:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:17Z" name: success-200-isvc-c3541-predictor.18c681dcaf03ffc9 namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: 2f5c555b-f678-46ea-854e-c96aef093e3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:44:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3541-predictor namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 6e8c0c05-a3c2-436d-baca-9dcca318404b kind: Event lastTimestamp: "2026-07-28T16:45:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:45:17Z" name: success-200-isvc-c3541-predictor.18c681dcaf75c719 namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: f3db0f5a-b86b-47f6-8df6-d0d876e994e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 71cfa773-bdfb-4e23-aedb-1c597a3e6ae7 kind: Event lastTimestamp: "2026-07-28T16:44:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:49Z" name: success-200-isvc-c3541.18c681d231e84d0d namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: cc811984-7042-46c7-8b28-4c77e391783a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 71cfa773-bdfb-4e23-aedb-1c597a3e6ae7 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-c3541": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3541": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541.18c681d23284fb0a namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: ebddfad3-cce1-4ad1-ad54-23ae4346aafd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 71cfa773-bdfb-4e23-aedb-1c597a3e6ae7 kind: Event lastTimestamp: "2026-07-28T16:44:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3541": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:02Z" name: success-200-isvc-c3541.18c681d232853b0a namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 3a83bdd9-9ef7-44b7-b6a4-46b311204325 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3541 namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 71cfa773-bdfb-4e23-aedb-1c597a3e6ae7 kind: Event lastTimestamp: "2026-07-28T16:44:49Z" message: InferenceService [success-200-isvc-c3541] is Ready metadata: creationTimestamp: "2026-07-28T16:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:44:49Z" name: success-200-isvc-c3541.18c681dd3e624de8 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 48e5ec40-f9d5-4654-954f-b7bc1faea6b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:34:37.025365Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7ba9d-5757fd8cc-68rtk to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:34:37Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c6814ea20ffa38 namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: ce0f218c-f2c3-41b5-9c2b-f62c330ac350 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-7ba9d-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:37Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c6814eaa8eac2a namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: d1aa271c-522e-45c7-a79b-583e90f1ee46 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:34:38Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c6814edf63adc2 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: de7ba4fa-0e92-4667-afd1-27113e7074dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7ba9d} kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c6814ee08e85e1 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 22278b27-85df-4f95-8beb-8df3766db370 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7ba9d} kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: 'Created container: switch-graph-7ba9d' metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c6814ee376b2df namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 28f4cbb1-e947-4e80-b9b6-880339746d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7ba9d} kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:34:38Z" message: Started container switch-graph-7ba9d metadata: creationTimestamp: "2026-07-28T16:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:38Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c6814ee3ed89f2 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 158004c1-2ed2-4409-9cf4-e9d502c758ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7ba9d} kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:42:51Z" message: Stopping container switch-graph-7ba9d metadata: creationTimestamp: "2026-07-28T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:51Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c681c1cdc3f88f namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 06561125-0116-42ee-921d-eaaecc664868 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7ba9d} kind: Pod name: switch-graph-7ba9d-5757fd8cc-68rtk namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b1e6379f-d775-4c57-bd47-d9e00c96f420 kind: Event lastTimestamp: "2026-07-28T16:43:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:43:19Z" name: switch-graph-7ba9d-5757fd8cc-68rtk.18c681c27d0cb963 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 66b6cc4d-790b-4b8b-8850-3b5ff30df8f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7ba9d-5757fd8cc namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 8859e313-cb7d-42ab-8663-4d0e7cd98d76 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: 'Created pod: switch-graph-7ba9d-5757fd8cc-68rtk' metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:37Z" name: switch-graph-7ba9d-5757fd8cc.18c6814ea197b26a namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 38628c80-3e2e-414d-a122-7fa0d40fcb89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 9858bb74-e36c-486d-9e0c-d154027785c7 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: Scaled up replica set switch-graph-7ba9d-5757fd8cc from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:37Z" name: switch-graph-7ba9d.18c6814ea0ba4970 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 7c92fcc6-8864-4c0c-8715-0c7fc18ac4e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 8fc52a59-a8d0-412c-bd4a-94e46579f6f2 kind: Event lastTimestamp: "2026-07-28T16:34:37Z" message: InferenceGraph [switch-graph-7ba9d] is Ready metadata: creationTimestamp: "2026-07-28T16:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:37Z" name: switch-graph-7ba9d.18c6814eaa74c83f namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 46de2cd1-bb63-491c-9e26-f4846f882d69 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: ae5c2409-b5b2-4df7-b85c-d19f2b9b215c kind: Event lastTimestamp: "2026-07-28T16:34:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:52Z" name: switch-graph-7ba9d.18c681521f77f1e5 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: e1a8f179-cce0-4635-a78a-b2fe1b38e6bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: ae5c2409-b5b2-4df7-b85c-d19f2b9b215c kind: Event lastTimestamp: "2026-07-28T16:34:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:52Z" name: switch-graph-7ba9d.18c681521fc30c56 namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 99173482-8974-4c15-a89f-5aaa14bfd5cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: ae5c2409-b5b2-4df7-b85c-d19f2b9b215c kind: Event lastTimestamp: "2026-07-28T16:35:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:07Z" name: switch-graph-7ba9d.18c681559e81b5cf namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: 7b222fdc-5cd1-4de4-a84d-b58c44662bf7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7ba9d namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: ae5c2409-b5b2-4df7-b85c-d19f2b9b215c kind: Event lastTimestamp: "2026-07-28T16:35:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:35:07Z" name: switch-graph-7ba9d.18c681559ee65659 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: 65a54b8c-397c-460a-b0a9-3329006d8bb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:33:26.681711Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b2ad5-6f6d6b5ff5-p57bx to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:33:26Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813e41410a5a namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 6ea3c9b4-15e6-4159-a4f8-89dec78e4fad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:26Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-b2ad5-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:26Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813e4be5076b namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 2c6e2b46-75ef-4355-9e9f-d71932a1c982 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:27Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:33:27Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813e7ed4f577 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d6bee074-5699-4ce4-9e39-d8e1dc81d065 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ad5} kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:27Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" metadata: creationTimestamp: "2026-07-28T16:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:27Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813e80103190 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: dfb50d9d-f7e2-4a89-a9c4-a35dfa2354df reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ad5} kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" in 2.139s (2.139s including waiting). Image size: 218226814 bytes.' metadata: creationTimestamp: "2026-07-28T16:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:29Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813eff955e5b namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 7dc4a3ab-7680-481f-bad7-e13de6fe672b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ad5} kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:29Z" message: 'Created container: switch-graph-b2ad5' metadata: creationTimestamp: "2026-07-28T16:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:29Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813f028925a4 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: e6c80b6e-f553-45d3-adf9-7edb11a4a7bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ad5} kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:29Z" message: Started container switch-graph-b2ad5 metadata: creationTimestamp: "2026-07-28T16:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:29Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c6813f02fa4f54 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: d01049f8-d992-48ac-bb98-28fe1fb88c65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ad5} kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:33:36Z" message: Stopping container switch-graph-b2ad5 metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:36Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c681409c3f041b namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 8010487a-a4c7-420a-8c13-e81331deedfd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:33:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2ad5} kind: Pod name: switch-graph-b2ad5-6f6d6b5ff5-p57bx namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 5d35afce-f447-4c44-a61f-38d89aa16812 kind: Event lastTimestamp: "2026-07-28T16:34:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:06Z" name: switch-graph-b2ad5-6f6d6b5ff5-p57bx.18c68141ab8b4a0c namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: ae2feb78-92d5-4e83-9182-20edfea99f0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b2ad5-6f6d6b5ff5 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 223ad40e-be58-4ba1-b2b0-a84d360a2e80 kind: Event lastTimestamp: "2026-07-28T16:33:26Z" message: 'Created pod: switch-graph-b2ad5-6f6d6b5ff5-p57bx' metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:26Z" name: switch-graph-b2ad5-6f6d6b5ff5.18c6813e40d7cec5 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: e635abb1-c8be-480e-a943-b993b53f45b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b2ad5 namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 4bfa5f35-a761-4c4d-bc08-f39610be6ea2 kind: Event lastTimestamp: "2026-07-28T16:33:26Z" message: Scaled up replica set switch-graph-b2ad5-6f6d6b5ff5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:26Z" name: switch-graph-b2ad5.18c6813e400f9b61 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: ace80af6-1863-4a25-8e0e-486429454e26 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b2ad5 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: 69201c1d-00c3-445e-965a-91b07e4b2528 kind: Event lastTimestamp: "2026-07-28T16:33:26Z" message: InferenceGraph [switch-graph-b2ad5] is Ready metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:26Z" name: switch-graph-b2ad5.18c6813e46023749 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: c9a9f769-4ac7-4153-b8a9-8ad73e49e370 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:55:16.919862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27297" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b9a8c-77584696f7-crzm9 to ip-10-0-139-46.ec2.internal metadata: creationTimestamp: "2026-07-28T16:55:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:55:16Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c6826f51889848 namespace: kserve-ci-e2e-test resourceVersion: "27303" uid: 2870bf89-e9d8-4dfe-a9cf-98d010428909 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85db6dd649-znt56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: "2026-07-28T16:55:17Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:55:17Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c6826f6aa261c2 namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 865499da-e8ea-4a77-bf01-c730d9facadd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9a8c} kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: "2026-07-28T16:55:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T16:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:17Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c6826f6bd9163a namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: a9097a7f-6dbb-46e0-98b3-aa29a41445e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9a8c} kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: "2026-07-28T16:55:17Z" message: 'Created container: switch-graph-b9a8c' metadata: creationTimestamp: "2026-07-28T16:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:17Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c6826f6e7d11ba namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: d600480d-643a-43f8-882e-5b0f9559a480 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9a8c} kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: "2026-07-28T16:55:17Z" message: Started container switch-graph-b9a8c metadata: creationTimestamp: "2026-07-28T16:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:55:17Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c6826f6eee7aef namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: fab54aec-eba0-4438-8fb0-c9e8df2eaad3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9a8c} kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: "2026-07-28T17:11:36Z" message: Stopping container switch-graph-b9a8c metadata: creationTimestamp: "2026-07-28T17:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:11:36Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c6835352730887 namespace: kserve-ci-e2e-test resourceVersion: "34170" uid: 0591500f-3a5d-4243-8bf3-3d7ec6fc7d0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:11:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9a8c} kind: Pod name: switch-graph-b9a8c-77584696f7-crzm9 namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 49c15256-d99c-4357-8f32-3a89d6d228e0 kind: Event lastTimestamp: "2026-07-28T17:12:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:12:04Z" name: switch-graph-b9a8c-77584696f7-crzm9.18c68353fc94a99f namespace: kserve-ci-e2e-test resourceVersion: "34443" uid: 48a50353-af52-4603-8f35-69da53e9d46d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-46.ec2.internal source: component: kubelet host: ip-10-0-139-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b9a8c-77584696f7 namespace: kserve-ci-e2e-test resourceVersion: "27294" uid: a6b48a88-6189-47d6-ae05-70844ad262d6 kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: 'Created pod: switch-graph-b9a8c-77584696f7-crzm9' metadata: creationTimestamp: "2026-07-28T16:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:16Z" name: switch-graph-b9a8c-77584696f7.18c6826f50fd50e8 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: 0b8c2c24-45da-48d6-ba88-e6dcdc539c3f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "27292" uid: f8d3539e-df21-408f-a4d1-1c29838c2c31 kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: Scaled up replica set switch-graph-b9a8c-77584696f7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:16Z" name: switch-graph-b9a8c.18c6826f50398711 namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: d79356a3-a454-4630-af09-498bb8f66653 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:55:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: 674c5130-0081-4ee4-9f10-370e298a97ab kind: Event lastTimestamp: "2026-07-28T16:55:16Z" message: InferenceGraph [switch-graph-b9a8c] is Ready metadata: creationTimestamp: "2026-07-28T16:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:55:16Z" name: switch-graph-b9a8c.18c6826f552b59b8 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 7be97dc0-5549-432f-836c-e1185683b75a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: 88ce2f37-5240-4b7c-b0f8-33cbeb5daa29 kind: Event lastTimestamp: "2026-07-28T16:55:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:46Z" name: switch-graph-b9a8c.18c68272cf1b5195 namespace: kserve-ci-e2e-test resourceVersion: "27592" uid: 88b3e1ca-c9de-44f3-9456-2f02a6d451eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:55:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b9a8c namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: 88ce2f37-5240-4b7c-b0f8-33cbeb5daa29 kind: Event lastTimestamp: "2026-07-28T16:55:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:55:46Z" name: switch-graph-b9a8c.18c68272cf65b9e5 namespace: kserve-ci-e2e-test resourceVersion: "27593" uid: 060d6b32-c726-49de-b136-96d7f53d214a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34451"