--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:31:16.889331Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-0d3ed-65bff8cdd9-2vns6 to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:31:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:31:16Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e011722c8d3 namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: e053a6ed-145b-49ff-ac9d-9a418ea94360 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:17Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-0d3ed-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:17Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e01211766e0 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 959d1e30-c647-441e-95af-697862589d02 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:17Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:17Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:31:17Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e01542ee190 namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: fa3e4d72-7754-4c8a-99a4-6bd28916e611 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0d3ed} kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:17Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e01556ff03d namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 26848d95-e46f-41ab-9e4c-3926136bb4ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0d3ed} kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:17Z" message: 'Created container: ensemble-graph-0d3ed' metadata: creationTimestamp: "2026-07-01T18:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:17Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e01585c9f87 namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: 44ad043f-0789-4d01-9ed9-a7981ac6253c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0d3ed} kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:17Z" message: Started container ensemble-graph-0d3ed metadata: creationTimestamp: "2026-07-01T18:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:17Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e0158c45e93 namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 81958fe5-e9fe-4751-8db5-65d58c7de6ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0d3ed} kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:26Z" message: Stopping container ensemble-graph-0d3ed metadata: creationTimestamp: "2026-07-01T18:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:26Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e036ffd16c2 namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: 2509b463-cac7-42d3-83e8-06b28d93518c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0d3ed} kind: Pod name: ensemble-graph-0d3ed-65bff8cdd9-2vns6 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 7d5fe37f-c403-4bb2-a3e0-23ad9c4173b8 kind: Event lastTimestamp: "2026-07-01T18:31:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:54Z" name: ensemble-graph-0d3ed-65bff8cdd9-2vns6.18be3e041b014a47 namespace: kserve-ci-e2e-test resourceVersion: "22487" uid: 071ffca9-825a-456c-8907-4a453d25aeb8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-0d3ed-65bff8cdd9 namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: affb737f-a09c-43ff-aeb1-5b140944463a kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'Created pod: ensemble-graph-0d3ed-65bff8cdd9-2vns6' metadata: creationTimestamp: "2026-07-01T18:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:16Z" name: ensemble-graph-0d3ed-65bff8cdd9.18be3e0116bdb747 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: ee4ff414-38c6-45fc-a57b-7be994d068be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: 9bbdb197-0023-460c-bfc6-e8b15a9ea27e kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: Scaled up replica set ensemble-graph-0d3ed-65bff8cdd9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:16Z" name: ensemble-graph-0d3ed.18be3e0116095b64 namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 8db5881a-9a3d-432c-a54c-a9f7a753bea8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: eb40be3a-a39e-4cb9-b581-40a59caeb3fe kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: InferenceGraph [ensemble-graph-0d3ed] is Ready metadata: creationTimestamp: "2026-07-01T18:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:16Z" name: ensemble-graph-0d3ed.18be3e011ad1d067 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 8041abe0-917c-48bc-b80e-f310d9029c65 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:32:27.177747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22961" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c1eb4-59675ff8b-b6nwr to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:32:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:32:27Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e1174a6df38 namespace: kserve-ci-e2e-test resourceVersion: "22966" uid: 57eaf530-d54c-4a81-b2bf-5479ddeb4176 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:32:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-c1eb4-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:27Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e11808631fa namespace: kserve-ci-e2e-test resourceVersion: "22979" uid: dd89b5a8-5389-4e03-94b4-2aee55ab9938 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:28Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22970" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:32:28Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:32:28Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e11b1d88e6a namespace: kserve-ci-e2e-test resourceVersion: "22986" uid: f25a711a-81b7-439f-9187-5daa2d301974 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1eb4} kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:32:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:28Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e11b3065647 namespace: kserve-ci-e2e-test resourceVersion: "22988" uid: 1d101cc4-50b4-458b-bf2b-15d0b65cc866 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1eb4} kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:32:28Z" message: 'Created container: ensemble-graph-c1eb4' metadata: creationTimestamp: "2026-07-01T18:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:28Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e11b5b8a7e0 namespace: kserve-ci-e2e-test resourceVersion: "22990" uid: 95170aaf-429c-44b0-b035-e4e5db4410fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1eb4} kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:32:28Z" message: Started container ensemble-graph-c1eb4 metadata: creationTimestamp: "2026-07-01T18:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:28Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e11b6217776 namespace: kserve-ci-e2e-test resourceVersion: "22991" uid: 33b4df32-3880-4137-a75d-17a24d3fde24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1eb4} kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: Stopping container ensemble-graph-c1eb4 metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:41Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e849b49b285 namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: 0f06e81b-663b-495d-b028-636bf6d5d634 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1eb4} kind: Pod name: ensemble-graph-c1eb4-59675ff8b-b6nwr namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 6df8fd9e-778f-4602-b75d-cc1436c7a0a4 kind: Event lastTimestamp: "2026-07-01T18:41:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:10Z" name: ensemble-graph-c1eb4-59675ff8b-b6nwr.18be3e8567af6b45 namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: 43b5f663-76dd-4159-8488-23f929a5a648 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c1eb4-59675ff8b namespace: kserve-ci-e2e-test resourceVersion: "22956" uid: c655516e-da8c-4f23-b928-fda50c5077fc kind: Event lastTimestamp: "2026-07-01T18:32:27Z" message: 'Created pod: ensemble-graph-c1eb4-59675ff8b-b6nwr' metadata: creationTimestamp: "2026-07-01T18:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:27Z" name: ensemble-graph-c1eb4-59675ff8b.18be3e11743db56f namespace: kserve-ci-e2e-test resourceVersion: "22962" uid: da30d5f3-b890-4fba-b9e4-2a34998f059c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22954" uid: 92ceb31c-7405-4763-9557-d4460913a617 kind: Event lastTimestamp: "2026-07-01T18:32:27Z" message: Scaled up replica set ensemble-graph-c1eb4-59675ff8b from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:27Z" name: ensemble-graph-c1eb4.18be3e117381d474 namespace: kserve-ci-e2e-test resourceVersion: "22958" uid: 347d9fbe-9132-474f-9624-54108fdbd935 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: 7248ccc0-1a5b-4cb1-9ba1-345c62b19000 kind: Event lastTimestamp: "2026-07-01T18:32:27Z" message: InferenceGraph [ensemble-graph-c1eb4] is Ready metadata: creationTimestamp: "2026-07-01T18:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:27Z" name: ensemble-graph-c1eb4.18be3e1178a588ac namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: e1ce00f4-d0ad-4ec9-ae0a-1dd71d6ebdb8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:32:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22955" uid: 738b80e4-8ac1-4ec9-a53e-7807a2cb9e8d kind: Event lastTimestamp: "2026-07-01T18:32:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:57Z" name: ensemble-graph-c1eb4.18be3e14f2553c1b namespace: kserve-ci-e2e-test resourceVersion: "23232" uid: efab9c0b-52ad-417f-b587-e86b91df1746 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:32:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22955" uid: 738b80e4-8ac1-4ec9-a53e-7807a2cb9e8d kind: Event lastTimestamp: "2026-07-01T18:32:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:57Z" name: ensemble-graph-c1eb4.18be3e14f2c64b47 namespace: kserve-ci-e2e-test resourceVersion: "23233" uid: 5dad1b3c-7308-4f8e-919c-c02b4262910d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:30:16.979023Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:30:16Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df324341075 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: a5687340-99c9-44d0-b192-2643fba7bd3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df33ea24a37 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 7434dc5c-ece0-4e84-b6ae-f3d0f8a505df reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df33feb76d3 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 99355870-95d6-48ee-9cf8-202afd4acc52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df34376f74a namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 987bcf03-65a5-4de9-87b2-54b8f05ae5d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df343f7ad0a namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 96054dbc-c588-47f3-88ea-1eff9bce8ceb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df344059913 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 124eb02a-012e-4180-9c5e-6cc59c259166 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df346e1792f namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 6ccbfaa5-d206-4a24-9fef-4857e6a06fa1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df34771105c namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 375bb5cf-57a6-4f01-b536-230b1fcb18c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:30:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:54Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3df3991d76b1 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: c080e5ba-66af-4808-9da9-b9adedb781c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3e0374e5b0d5 namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 11792bfd-959a-4101-b5bb-347b5b5d3a47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3e0374e66154 namespace: kserve-ci-e2e-test resourceVersion: "22150" uid: 7fe43829-e4de-4454-81ee-489e91fd1353 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 48dac5a4-feb9-4052-9d7d-120cb2555d0e kind: Event lastTimestamp: "2026-07-01T18:31:29Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:29Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd.18be3e042139346e namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: 7854e518-628d-4865-ac32-c8fa8088b362 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0d3ed-predictor-6d6c58bcc5 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 666568c6-8064-42e0-9d56-1746cda5a776 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: 'Created pod: error-404-isvc-0d3ed-predictor-6d6c58bcc5-545nd' metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:16Z" name: error-404-isvc-0d3ed-predictor-6d6c58bcc5.18be3df323ccbfc2 namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: be7133e9-7bb9-473c-b7c6-4618504c7c3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 1bb3d7ac-b691-4c26-9bad-2c868aa5f6ab kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: Scaled up replica set error-404-isvc-0d3ed-predictor-6d6c58bcc5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:16Z" name: error-404-isvc-0d3ed-predictor.18be3df32186d0f1 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: c501cae5-96f0-404c-90ca-9f74f119e968 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 096dc5a4-9c00-4487-a736-6f55d533d3f3 kind: Event lastTimestamp: "2026-07-01T18:30:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:46Z" name: error-404-isvc-0d3ed-predictor.18be3df6a039ec84 namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 02cdc1e2-4104-49ba-8c26-0121eaf90722 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 096dc5a4-9c00-4487-a736-6f55d533d3f3 kind: Event lastTimestamp: "2026-07-01T18:30:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:46Z" name: error-404-isvc-0d3ed-predictor.18be3df6a0856020 namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 3f817d74-f245-4845-bb74-3a63fb5676b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 096dc5a4-9c00-4487-a736-6f55d533d3f3 kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:16Z" name: error-404-isvc-0d3ed-predictor.18be3dfd9e30086c namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: e4658660-e670-4970-befe-493c0cf85bd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 096dc5a4-9c00-4487-a736-6f55d533d3f3 kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:17Z" name: error-404-isvc-0d3ed-predictor.18be3dfd9ea97c7b namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 05211409-15af-441a-8557-feec67b20859 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 8d22924d-a004-474b-85aa-7d2f84af3051 kind: Event lastTimestamp: "2026-07-01T18:31:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:05Z" name: error-404-isvc-0d3ed.18be3df32236489e namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: 958b263f-fb85-4ce4-840f-fdd9e864f38f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 8d22924d-a004-474b-85aa-7d2f84af3051 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-0d3ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0d3ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:30:16Z" name: error-404-isvc-0d3ed.18be3df322e54163 namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 88f5f459-fd88-49e6-9e9e-a07b0a8216c8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 8d22924d-a004-474b-85aa-7d2f84af3051 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0d3ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:30:16Z" name: error-404-isvc-0d3ed.18be3df322e5a3aa namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 1fb4dcd6-7130-4613-8d61-5469fc29de21 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 8d22924d-a004-474b-85aa-7d2f84af3051 kind: Event lastTimestamp: "2026-07-01T18:31:05Z" message: InferenceService [error-404-isvc-0d3ed] is Ready metadata: creationTimestamp: "2026-07-01T18:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:05Z" name: error-404-isvc-0d3ed.18be3dfe570c389a namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 17532df3-71ab-494c-92f1-9e817c31ec25 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:41:52.323366Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27754" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e9509f4b9cc namespace: kserve-ci-e2e-test resourceVersion: "27758" uid: d0a88615-69e4-4c68-8d9f-156db3cb638f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1f8d1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e951252aed4 namespace: kserve-ci-e2e-test resourceVersion: "27804" uid: 9b4bac31-f5f9-433a-a13d-edbf5d473d6a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27764" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e957d81fdd0 namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: 4b74a224-b7ce-46f2-b86a-53ecc81d722b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e957ee3c419 namespace: kserve-ci-e2e-test resourceVersion: "27822" uid: 9bc0236a-fcf6-4217-8b1f-7fbd5aed6658 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e95825eeec1 namespace: kserve-ci-e2e-test resourceVersion: "27824" uid: f3a9d011-585a-4491-ab4c-786c20f7d9c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e958362bd71 namespace: kserve-ci-e2e-test resourceVersion: "27825" uid: a29e44ce-a9e7-4ce9-8b15-df608167407d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e95836dbaf8 namespace: kserve-ci-e2e-test resourceVersion: "27826" uid: 7ea2c6b5-ef07-4942-8ef0-467b72492603 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e958606f3b6 namespace: kserve-ci-e2e-test resourceVersion: "27828" uid: 7f81b27a-dd09-409a-926e-04c754697cc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e95867c8095 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 87ba1b44-2688-4e6b-a077-69a63b11c223 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:42:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:40Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3e958ddb423a namespace: kserve-ci-e2e-test resourceVersion: "28271" uid: 31966f3d-0907-4243-b556-ab9ee20a3dd9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:51:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:51:07Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3f163b92cbb6 namespace: kserve-ci-e2e-test resourceVersion: "31904" uid: 55b6c2bf-9f79-4069-a2e3-6c02b36dd519 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 71a450af-7792-4cdf-a498-22f66f8e38ff kind: Event lastTimestamp: "2026-07-01T18:51:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:51:07Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj.18be3f163b935d95 namespace: kserve-ci-e2e-test resourceVersion: "31905" uid: 68eeef82-d8bb-4882-a6ae-4a2a3a4b50ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1f8d1-predictor-9c8b6b447 namespace: kserve-ci-e2e-test resourceVersion: "27747" uid: 55c3255b-910e-428f-8296-604a8d23c996 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'Created pod: error-404-isvc-1f8d1-predictor-9c8b6b447-5pwvj' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-1f8d1-predictor-9c8b6b447.18be3e9509a34909 namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: 68d1b6cc-4be0-4d27-bbae-d35f96193fd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27744" uid: 89d84e10-b9c4-4291-aa90-89fd9ae28726 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Scaled up replica set error-404-isvc-1f8d1-predictor-9c8b6b447 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-1f8d1-predictor.18be3e9507bd6717 namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: 52fc0539-a7eb-4353-a27b-541705bd8fcd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27746" uid: 8f5d56fa-9db5-40a7-ab58-e3e55496388d kind: Event lastTimestamp: "2026-07-01T18:42:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:37Z" name: error-404-isvc-1f8d1-predictor.18be3e98862eddab namespace: kserve-ci-e2e-test resourceVersion: "28204" uid: 5b2ddd33-4646-44f3-8fe2-7e2c309d84f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27746" uid: 8f5d56fa-9db5-40a7-ab58-e3e55496388d kind: Event lastTimestamp: "2026-07-01T18:42:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:37Z" name: error-404-isvc-1f8d1-predictor.18be3e98869da942 namespace: kserve-ci-e2e-test resourceVersion: "28205" uid: 3a2c4992-3927-442e-ad2b-b6b92da90aa6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27945" uid: 8f5d56fa-9db5-40a7-ab58-e3e55496388d kind: Event lastTimestamp: "2026-07-01T18:43:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:22Z" name: error-404-isvc-1f8d1-predictor.18be3ea302ed7138 namespace: kserve-ci-e2e-test resourceVersion: "28642" uid: 2ad84a61-4582-49d3-836c-48cf5793c8c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27945" uid: 8f5d56fa-9db5-40a7-ab58-e3e55496388d kind: Event lastTimestamp: "2026-07-01T18:43:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:22Z" name: error-404-isvc-1f8d1-predictor.18be3ea303622014 namespace: kserve-ci-e2e-test resourceVersion: "28643" uid: 6faea9ba-8476-4052-b61d-79b2e90a4e2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "27730" uid: e7be16db-3b92-4fd5-8201-06668845a05f kind: Event lastTimestamp: "2026-07-01T18:42:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:50Z" name: error-404-isvc-1f8d1.18be3e9507b8431e namespace: kserve-ci-e2e-test resourceVersion: "28360" uid: 7fe510ab-2733-4ecd-b47d-c59944625e18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "27730" uid: e7be16db-3b92-4fd5-8201-06668845a05f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-1f8d1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f8d1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-1f8d1.18be3e95080f66fc namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: 8e58e412-fdcc-47a2-a965-f48489a5a7ba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "27730" uid: e7be16db-3b92-4fd5-8201-06668845a05f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f8d1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-1f8d1.18be3e95080fab5f namespace: kserve-ci-e2e-test resourceVersion: "27752" uid: dd1988ad-ff43-4c50-991f-a39dfeacae5b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "28357" uid: e7be16db-3b92-4fd5-8201-06668845a05f kind: Event lastTimestamp: "2026-07-01T18:42:50Z" message: InferenceService [error-404-isvc-1f8d1] is Ready metadata: creationTimestamp: "2026-07-01T18:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:50Z" name: error-404-isvc-1f8d1.18be3ea29cc6d430 namespace: kserve-ci-e2e-test resourceVersion: "28358" uid: 86102abe-1fca-43a9-9133-a497d8e57dde reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:21:02.167431Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d71f6db0242 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 56344852-5cc7-4ca3-91c0-256f01a152a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2588a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d7201d98af0 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 4860ab7b-7048-45c7-8856-f3fca4aeaa92 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d72346edf09 namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: e5daf947-985b-46cd-a72b-43960c102b4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d7235bde176 namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: 65336972-0995-4262-b6af-fee57a85e582 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d72390bf98e namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: dc658064-9139-455c-a5f9-14deb4d31ddf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d72398158ab namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: a40d8b4f-fc9f-49f3-8df3-d8d3332e7899 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d72398c0082 namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: 9e4ea563-8606-4489-af2c-f85b33787881 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d723c62bad5 namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: 4ec3f7f6-5933-460d-a5cd-7c3d7ace7d51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d723cddee0d namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: f998dd67-84e0-4151-8f74-7b589116f80d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:21:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:40Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3d729a6a1023 namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: dd1ae8b6-066a-437d-a787-7c8cfb4dd314 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3df3281f4c65 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: bdc124f2-4cd2-435a-a870-113b54288d93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6d05b78c-7dca-4f07-b37a-007729f90494 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8.18be3df3281ff46c namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 86d8a4f1-fe2d-485e-a99b-2de540028c03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2588a-predictor-75c88dcf4c namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: d48e4307-adcf-470a-b0d7-a328dd659651 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'Created pod: error-404-isvc-2588a-predictor-75c88dcf4c-qp2m8' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-2588a-predictor-75c88dcf4c.18be3d71f66561c1 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: d512d282-5da0-41bb-a06e-ddd1833021ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: edcb02ad-3e34-43a2-965c-1c898113b2b1 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: Scaled up replica set error-404-isvc-2588a-predictor-75c88dcf4c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-2588a-predictor.18be3d71f4b6751f namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 7d4f1fdc-f8aa-4b2a-8487-19aba4d386ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:21:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 7b805a25-3e5e-4c5f-a969-1c3b9551aff3 kind: Event lastTimestamp: "2026-07-01T18:21:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:47Z" name: error-404-isvc-2588a-predictor.18be3d75735bc02e namespace: kserve-ci-e2e-test resourceVersion: "17361" uid: 796376e0-e52d-4210-8d81-733534cf1720 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:21:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 7b805a25-3e5e-4c5f-a969-1c3b9551aff3 kind: Event lastTimestamp: "2026-07-01T18:21:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:47Z" name: error-404-isvc-2588a-predictor.18be3d7573d82a4f namespace: kserve-ci-e2e-test resourceVersion: "17362" uid: a2516d7d-8ee5-47d3-96a5-a2492f35973b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 7b805a25-3e5e-4c5f-a969-1c3b9551aff3 kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:17Z" name: error-404-isvc-2588a-predictor.18be3d7ff04edbf2 namespace: kserve-ci-e2e-test resourceVersion: "17872" uid: 1b9a9cb3-aea3-41cb-ad0f-7fafaae5346c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 7b805a25-3e5e-4c5f-a969-1c3b9551aff3 kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:17Z" name: error-404-isvc-2588a-predictor.18be3d7ff0bdc07e namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: 901f63d3-4bcb-4997-904e-704d194eb604 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 50797785-d1da-483b-b1b6-73763e2a0ca4 kind: Event lastTimestamp: "2026-07-01T18:21:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:51Z" name: error-404-isvc-2588a.18be3d71f4bde1d7 namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: e35025b6-2e6f-4f58-9edc-a1095ace7ce7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 50797785-d1da-483b-b1b6-73763e2a0ca4 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-2588a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2588a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-2588a.18be3d71f5225727 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: a2464647-f31e-49f4-9c09-bcb44b73a51c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 50797785-d1da-483b-b1b6-73763e2a0ca4 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2588a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-2588a.18be3d71f522a7b8 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 2a2f2137-74c8-4db4-b88c-9de5c7ce9b55 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: 50797785-d1da-483b-b1b6-73763e2a0ca4 kind: Event lastTimestamp: "2026-07-01T18:21:51Z" message: InferenceService [error-404-isvc-2588a] is Ready metadata: creationTimestamp: "2026-07-01T18:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:51Z" name: error-404-isvc-2588a.18be3d7d559c5a65 namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: 0a185346-7241-467d-a024-2f47a6d26f0d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:32:23.075805Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22864" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-293b3-predictor-864bd98777-svbdl to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e108028334c namespace: kserve-ci-e2e-test resourceVersion: "22868" uid: 30df32b9-8dce-4a08-b168-a081eb4051a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22874" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e1099dee9f6 namespace: kserve-ci-e2e-test resourceVersion: "22892" uid: dc3a7585-8c1b-4fa3-a58f-b747dc66c1e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e109b2f0c10 namespace: kserve-ci-e2e-test resourceVersion: "22894" uid: 197350bb-c186-4f08-aade-871b0f484195 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e109e99188c namespace: kserve-ci-e2e-test resourceVersion: "22897" uid: c14d9ade-86ee-4a4f-825f-9e3ffc0c0815 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e109f0fb29d namespace: kserve-ci-e2e-test resourceVersion: "22898" uid: 5b6f699b-89b8-4e6d-bc7b-12a3a801e42c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e109f1991e9 namespace: kserve-ci-e2e-test resourceVersion: "22899" uid: fadbd7ab-24d0-4dca-a142-27b38fcaeb66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e10a1e10152 namespace: kserve-ci-e2e-test resourceVersion: "22900" uid: 6171749d-926d-4673-a45d-efcb4c7c0e4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e10a25c5f17 namespace: kserve-ci-e2e-test resourceVersion: "22901" uid: 2fae14ab-73e1-489a-9549-850a6f701183 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:41:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:41Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e110d9ebaaf namespace: kserve-ci-e2e-test resourceVersion: "27537" uid: f1249762-714e-4f4e-a4ee-d3432cd363a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e91a9c8843d namespace: kserve-ci-e2e-test resourceVersion: "27397" uid: cfc58ed2-50ed-4048-978f-33d704529874 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e91a9c9495a namespace: kserve-ci-e2e-test resourceVersion: "27401" uid: fc1b22b3-f986-4a55-b1c1-d095bf238530 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-293b3-predictor-864bd98777-svbdl namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: cac7e284-aea5-4817-b2ea-927dbc78868a kind: Event lastTimestamp: "2026-07-01T18:41:41Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:41Z" name: error-404-isvc-293b3-predictor-864bd98777-svbdl.18be3e92821f188b namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: b689eccb-d471-42d8-94d7-9f4b6d61b482 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-293b3-predictor-864bd98777 namespace: kserve-ci-e2e-test resourceVersion: "22853" uid: 7a01adc3-9909-49a0-b50b-2f14bd9b484e kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'Created pod: error-404-isvc-293b3-predictor-864bd98777-svbdl' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor-864bd98777.18be3e107fe1f051 namespace: kserve-ci-e2e-test resourceVersion: "22866" uid: 535e6204-4fbb-4d13-a990-0a9ae99b2692 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "22850" uid: 23cdc155-fdf1-459e-9c7e-cbdfeee8dcbc kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Scaled up replica set error-404-isvc-293b3-predictor-864bd98777 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3-predictor.18be3e107df2b290 namespace: kserve-ci-e2e-test resourceVersion: "22855" uid: 4a55cbb5-0193-4bd2-80c7-d4ceaf245c82 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "22852" uid: adc2f0f5-54f1-4ab2-b794-2211a67a28ff kind: Event lastTimestamp: "2026-07-01T18:32:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:38Z" name: error-404-isvc-293b3-predictor.18be3e13fcf5e88d namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 4f9b0be6-3b84-4c9e-be29-45abd5e43fc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "22852" uid: adc2f0f5-54f1-4ab2-b794-2211a67a28ff kind: Event lastTimestamp: "2026-07-01T18:32:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:38Z" name: error-404-isvc-293b3-predictor.18be3e13fd78508d namespace: kserve-ci-e2e-test resourceVersion: "23076" uid: 2d65cb64-de91-4233-b257-9cbd407b21e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23075" uid: adc2f0f5-54f1-4ab2-b794-2211a67a28ff kind: Event lastTimestamp: "2026-07-01T18:33:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:38Z" name: error-404-isvc-293b3-predictor.18be3e177c5b5b84 namespace: kserve-ci-e2e-test resourceVersion: "23592" uid: d896beb3-8b29-4af5-bab0-42295bb8c935 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23075" uid: adc2f0f5-54f1-4ab2-b794-2211a67a28ff kind: Event lastTimestamp: "2026-07-01T18:33:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:38Z" name: error-404-isvc-293b3-predictor.18be3e177cf41292 namespace: kserve-ci-e2e-test resourceVersion: "23593" uid: 6410fc37-a41e-49af-a4f6-b4203bf8bd4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-293b3 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: c0d4fbd3-4e89-47f5-bd15-7ceb0d3eecd4 kind: Event lastTimestamp: "2026-07-01T18:33:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:11Z" name: error-404-isvc-293b3.18be3e107df1dd0b namespace: kserve-ci-e2e-test resourceVersion: "23360" uid: 93c7d6e4-31fc-4b10-a07b-a439f594ed07 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-293b3 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: c0d4fbd3-4e89-47f5-bd15-7ceb0d3eecd4 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-293b3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-293b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3.18be3e107e68097d namespace: kserve-ci-e2e-test resourceVersion: "22859" uid: ded44a5b-eb71-4e1f-9615-db1485ff4872 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-293b3 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: c0d4fbd3-4e89-47f5-bd15-7ceb0d3eecd4 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-293b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:23Z" name: error-404-isvc-293b3.18be3e107e685e5e namespace: kserve-ci-e2e-test resourceVersion: "22862" uid: e807d810-5a54-4d2f-be93-7cac68085191 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23358" uid: c0d4fbd3-4e89-47f5-bd15-7ceb0d3eecd4 kind: Event lastTimestamp: "2026-07-01T18:33:11Z" message: InferenceService [error-404-isvc-293b3] is Ready metadata: creationTimestamp: "2026-07-01T18:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:11Z" name: error-404-isvc-293b3.18be3e1bc9576b2c namespace: kserve-ci-e2e-test resourceVersion: "23359" uid: a34cffd5-cefd-401a-8013-7057f1d49dab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:19:27.947837Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4d812-predictor-cf946d449-cpmt9 to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:19:27Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5c06ee0daf namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 4a9f19b3-c857-4602-8883-3f136c742005 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:19:28Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5c20ae43bd namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 2c6a506e-1534-491f-8847-67a25dd62cf4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5c21f0e7a6 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 8c57b3ae-5f3c-4372-bc67-536d682db13c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" in 12.743s (12.743s including waiting). Image size: 1334689372 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5f1987f30d namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 23447c36-8221-4044-b7d9-9b776d6f6402 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5f1cc70ad6 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 3672e3ef-34c9-4889-a555-d785343be68c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5f1d3d96d9 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: b7f1938e-6497-4055-98d6-747a570403a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5f1d4afbf5 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 67e511ae-0269-458e-bc64-c64685febb6f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.078s (3.078s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:44Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5fd4c782f8 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 4642b1b4-0fee-4d14-8f4a-d6932aa0fcbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:44Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5fd8acd416 namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: b0db9132-6669-408e-8bd1-daa5684c9c7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:19:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:44Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5fd92a4e81 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: dd5ba6e9-201c-4cfa-a625-6a0cbe19529c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:20:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:20Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d5fe8df334c namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: 977100f7-d6fd-4c90-b1db-bbaccad5673d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d71fa45688a namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 4d0fe8bf-364b-475a-ac13-e7ca4cc69fc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:02Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d71fa45cde2 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: e511bcb8-311c-41ec-9fa0-8e9d9eccca96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d812-predictor-cf946d449-cpmt9 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8b988bd0-5e2e-44de-80d1-3bde2076b377 kind: Event lastTimestamp: "2026-07-01T18:21:05Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:05Z" name: error-404-isvc-4d812-predictor-cf946d449-cpmt9.18be3d72c4ff4420 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 14fa76e2-3ac3-4631-8b71-9e70e0ca6c85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4d812-predictor-cf946d449 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 61987e6c-8a22-4596-8259-8c7f575f1e90 kind: Event lastTimestamp: "2026-07-01T18:19:27Z" message: 'Created pod: error-404-isvc-4d812-predictor-cf946d449-cpmt9' metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:27Z" name: error-404-isvc-4d812-predictor-cf946d449.18be3d5c0648b50a namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 6e5a65a9-a043-43f8-a7b8-b472ae7fa2b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: d555c3dc-d21b-4d76-b3dc-7ef0026ce1d1 kind: Event lastTimestamp: "2026-07-01T18:19:27Z" message: Scaled up replica set error-404-isvc-4d812-predictor-cf946d449 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:27Z" name: error-404-isvc-4d812-predictor.18be3d5c03d4f6f3 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 76709a32-9334-4403-a6af-4903e8211e06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 7d1d20bc-3b88-4681-9bcc-bb68c3b20356 kind: Event lastTimestamp: "2026-07-01T18:19:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:42Z" name: error-404-isvc-4d812-predictor.18be3d5f825c4f71 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: bb2a6f04-8179-40e1-b348-10809a7df780 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 7d1d20bc-3b88-4681-9bcc-bb68c3b20356 kind: Event lastTimestamp: "2026-07-01T18:19:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:42Z" name: error-404-isvc-4d812-predictor.18be3d5f82c74deb namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: b12037f6-88f2-48a7-96db-a9166b9cffc4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 7d1d20bc-3b88-4681-9bcc-bb68c3b20356 kind: Event lastTimestamp: "2026-07-01T18:20:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:42Z" name: error-404-isvc-4d812-predictor.18be3d63018afbf5 namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: a01141bb-3273-4815-8508-e929efd909be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 7d1d20bc-3b88-4681-9bcc-bb68c3b20356 kind: Event lastTimestamp: "2026-07-01T18:20:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:42Z" name: error-404-isvc-4d812-predictor.18be3d6301fc20ee namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 029b7f37-1a04-49e2-afd9-c7161318d682 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4d812 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: aacb0bc7-a61f-4cf4-97a3-ef323fef52fa kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:30Z" name: error-404-isvc-4d812.18be3d5c03df08b9 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 818302e7-7803-4182-80ab-6b6701645181 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4d812 namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: aacb0bc7-a61f-4cf4-97a3-ef323fef52fa kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: InferenceService [error-404-isvc-4d812] is Ready metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:30Z" name: error-404-isvc-4d812.18be3d6aa44ccaa7 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: f22ba6e4-c814-4420-afe8-220ad548d0c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:41:37.987532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27456" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-60085-predictor-6857fddfd9-jg6vd to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91b37941ad namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 725ef4b7-c670-49ac-8fc6-bd8231e70062 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-60085-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91bd57d309 namespace: kserve-ci-e2e-test resourceVersion: "27472" uid: 3b9c39ed-a02a-4dc0-8b39-1c52989e7e32 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91f1067f14 namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 06d29c5e-9604-41ba-a7f2-03b3d769e4fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91f27f141d namespace: kserve-ci-e2e-test resourceVersion: "27499" uid: 7b0600ef-fbd8-4ebc-8ad4-7f28d984e0b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91f6d67c7a namespace: kserve-ci-e2e-test resourceVersion: "27502" uid: 7df38d22-1b31-4efc-bfff-33c5460ddfe1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91f751c80b namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: fa356f9b-e37c-4058-83ad-774d7964799e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91f75cc4a3 namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: b5687638-8522-44dc-af34-2700ec140906 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91fac06b42 namespace: kserve-ci-e2e-test resourceVersion: "27505" uid: af8c4faf-afc9-48b1-971a-a6b2569ebd57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:41:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:39Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e91fb396b7f namespace: kserve-ci-e2e-test resourceVersion: "27506" uid: ebd2b341-db23-4565-a9c4-d7b52a4956d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:41:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:42:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:25Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3e920bcfaa1a namespace: kserve-ci-e2e-test resourceVersion: "28112" uid: e4d876f4-fd2e-4b6f-ae67-c9268c95fc3f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:58:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:57Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3f83b3cb735e namespace: kserve-ci-e2e-test resourceVersion: "35175" uid: ec1286e3-e5d0-43cc-b2a1-1e7a4c50d001 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:58:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:57Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3f83b3cc14b5 namespace: kserve-ci-e2e-test resourceVersion: "35176" uid: 0c8ff5ca-1b45-4ff5-bb10-34b710a2bafb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 9a6ee025-2df3-42de-9e35-92c6163c8229 kind: Event lastTimestamp: "2026-07-01T18:59:00Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:00Z" name: error-404-isvc-60085-predictor-6857fddfd9-jg6vd.18be3f846c69178e namespace: kserve-ci-e2e-test resourceVersion: "35232" uid: 6d147ab8-24be-4e0a-a2f4-397a1470ceef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-60085-predictor-6857fddfd9 namespace: kserve-ci-e2e-test resourceVersion: "27447" uid: 093dd458-5e06-44bf-a198-32822fca60c9 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: 'Created pod: error-404-isvc-60085-predictor-6857fddfd9-jg6vd' metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-60085-predictor-6857fddfd9.18be3e91b30eac2d namespace: kserve-ci-e2e-test resourceVersion: "27457" uid: 996032f5-7d16-427d-8c61-7ae732af83ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: 669a8576-6a91-407d-af22-cf7166906402 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Scaled up replica set error-404-isvc-60085-predictor-6857fddfd9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-60085-predictor.18be3e91b165056f namespace: kserve-ci-e2e-test resourceVersion: "27450" uid: 41dc7f94-883d-46e7-af87-00ff81489229 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: 1d7e3174-5f21-43d6-96b4-3952de1b96ef kind: Event lastTimestamp: "2026-07-01T18:42:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:22Z" name: error-404-isvc-60085-predictor.18be3e952fe2bea8 namespace: kserve-ci-e2e-test resourceVersion: "28086" uid: 17f73f5f-efce-45a5-9d85-74e8686313e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: 1d7e3174-5f21-43d6-96b4-3952de1b96ef kind: Event lastTimestamp: "2026-07-01T18:42:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:23Z" name: error-404-isvc-60085-predictor.18be3e95304e4e2d namespace: kserve-ci-e2e-test resourceVersion: "28087" uid: ea5f6736-7728-43c8-9088-4f8c3be53792 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27802" uid: 1d7e3174-5f21-43d6-96b4-3952de1b96ef kind: Event lastTimestamp: "2026-07-01T18:43:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:08Z" name: error-404-isvc-60085-predictor.18be3e9fad24a454 namespace: kserve-ci-e2e-test resourceVersion: "28543" uid: cb132497-3645-4d0a-b1af-1814263fb1a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27802" uid: 1d7e3174-5f21-43d6-96b4-3952de1b96ef kind: Event lastTimestamp: "2026-07-01T18:43:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:08Z" name: error-404-isvc-60085-predictor.18be3e9fad9f2905 namespace: kserve-ci-e2e-test resourceVersion: "28544" uid: 32053591-81ba-4ae8-ab90-43bf7f810d5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60085 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 067d2a8e-ffa4-445d-be8b-ded1553fdce1 kind: Event lastTimestamp: "2026-07-01T18:42:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:35Z" name: error-404-isvc-60085.18be3e91b178307e namespace: kserve-ci-e2e-test resourceVersion: "28188" uid: 41fa99c4-541e-45a1-aa59-a08832368d18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60085 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 067d2a8e-ffa4-445d-be8b-ded1553fdce1 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-60085": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60085": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-60085.18be3e91b1e25e51 namespace: kserve-ci-e2e-test resourceVersion: "27453" uid: ac5b9626-f098-40ce-ade8-b481fa6eae02 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60085 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 067d2a8e-ffa4-445d-be8b-ded1553fdce1 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60085": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:37Z" name: error-404-isvc-60085.18be3e91b1e29fbb namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 7f2db55a-c314-47b8-ae6c-04a5e406d6ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60085 namespace: kserve-ci-e2e-test resourceVersion: "28186" uid: 067d2a8e-ffa4-445d-be8b-ded1553fdce1 kind: Event lastTimestamp: "2026-07-01T18:42:35Z" message: InferenceService [error-404-isvc-60085] is Ready metadata: creationTimestamp: "2026-07-01T18:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:35Z" name: error-404-isvc-60085.18be3e9f1b1d12cc namespace: kserve-ci-e2e-test resourceVersion: "28187" uid: 4e123cc7-8492-418e-be89-983d07b12b07 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:31:12.827893Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9b590-predictor-954bbfb5c-65n5p to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e00250e26f9 namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: 02717f3e-3af9-42de-b154-4c51627d7904 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e003eda80b3 namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: d2210bcf-b694-4524-a3bb-68e31984d2ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e004028a280 namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: f2b3c62e-9dc4-4883-9d04-88ff31910d45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e0042d00e2c namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 2ac705e1-0a34-4f35-a186-29bb9d73e8c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e004349c970 namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: 3bb89c5e-096f-4aa1-ab0c-309c4dc8d721 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e0043543bd6 namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 9b2dc62c-e80b-4ae3-a9d0-9241c41c6201 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e0046e5cd87 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 31a44c1e-fc44-430e-9b1c-1f593585d32a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e00476b3791 namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: 655e30d7-278c-4da7-920a-4dfe7880c459 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:31:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:51Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e00b0897a46 namespace: kserve-ci-e2e-test resourceVersion: "22459" uid: 969e3b3a-7d9f-4828-811b-0a4a461f2c4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:22Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e10770ef8bf namespace: kserve-ci-e2e-test resourceVersion: "22809" uid: 27878729-3fa3-4d49-aac1-aa335023f588 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:22Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e10770f617c namespace: kserve-ci-e2e-test resourceVersion: "22814" uid: 9048bd81-49c5-4c83-aa08-5bbae78c02be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 18bcbb28-c219-45dc-b867-abcf7f139ec3 kind: Event lastTimestamp: "2026-07-01T18:32:26Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:26Z" name: error-404-isvc-9b590-predictor-954bbfb5c-65n5p.18be3e1138c049d9 namespace: kserve-ci-e2e-test resourceVersion: "22938" uid: a624491b-8ac1-45c6-9cdf-b84ae53eb0f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9b590-predictor-954bbfb5c namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: c65c9859-c2a9-4e2d-bf99-53d6eae4b9b5 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'Created pod: error-404-isvc-9b590-predictor-954bbfb5c-65n5p' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-9b590-predictor-954bbfb5c.18be3e0024a956b9 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: a9ef50be-83bd-4d17-9a19-87dcc75078f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 8206aec7-0d45-434c-9370-42738583c068 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Scaled up replica set error-404-isvc-9b590-predictor-954bbfb5c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-9b590-predictor.18be3e0022f9977c namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 9d6aa9f3-d79f-48f7-86a2-bffc6f542dab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: c6961a6d-d2fc-4896-ac5f-8296c55263d2 kind: Event lastTimestamp: "2026-07-01T18:31:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:42Z" name: error-404-isvc-9b590-predictor.18be3e03a1973345 namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: 2f14774e-91b6-4c71-b31a-a31e40929a79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: c6961a6d-d2fc-4896-ac5f-8296c55263d2 kind: Event lastTimestamp: "2026-07-01T18:31:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:42Z" name: error-404-isvc-9b590-predictor.18be3e03a1fe721c namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: 2eb6263f-9e8c-45a4-8a23-7ab324cd9baf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "22241" uid: c6961a6d-d2fc-4896-ac5f-8296c55263d2 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:12Z" name: error-404-isvc-9b590-predictor.18be3e0a9fc078bc namespace: kserve-ci-e2e-test resourceVersion: "22670" uid: b46af101-6fe2-4ded-ac08-877e5159d5f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "22241" uid: c6961a6d-d2fc-4896-ac5f-8296c55263d2 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:12Z" name: error-404-isvc-9b590-predictor.18be3e0aa043ef7f namespace: kserve-ci-e2e-test resourceVersion: "22672" uid: 45628306-961c-4d47-a819-f2c56586ac6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: c395095a-f46b-40ff-9297-9c7c11ff776b kind: Event lastTimestamp: "2026-07-01T18:32:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:01Z" name: error-404-isvc-9b590.18be3e00230075de namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: 65a4ae07-9b55-4d3f-b488-db941cad3cb9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: c395095a-f46b-40ff-9297-9c7c11ff776b kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-9b590": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9b590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-9b590.18be3e00236f3f5d namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: aabd9dfb-2b1d-4d8d-b485-34b905026dee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: c395095a-f46b-40ff-9297-9c7c11ff776b kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9b590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-9b590.18be3e00236f6d55 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: b24c2802-36ad-4fcc-8307-23ca040bd407 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: c395095a-f46b-40ff-9297-9c7c11ff776b kind: Event lastTimestamp: "2026-07-01T18:32:01Z" message: InferenceService [error-404-isvc-9b590] is Ready metadata: creationTimestamp: "2026-07-01T18:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:01Z" name: error-404-isvc-9b590.18be3e0b6bc0ffb9 namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: a02919fc-2f73-4f69-86b3-8c297e5d55e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:31:27.225434Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e037f3739c5 namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 26efe3bf-5d10-46c5-bdc3-e5a240953836 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22212" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e03992052ae namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 1f3cfd91-6503-4185-bea9-dd9af70322c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e039aa90af5 namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 9a03da51-b341-4e30-bbb4-be047ab231d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e039dce14b9 namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: eba6b707-6422-4af4-9d04-80e1b49f8c25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e039e458095 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 7d50778b-8f91-4d4c-9e46-ba4815c23c05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e039e4fe686 namespace: kserve-ci-e2e-test resourceVersion: "22239" uid: ff43f1f7-4936-4f03-98f4-69ea6da997f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e03a244555d namespace: kserve-ci-e2e-test resourceVersion: "22243" uid: 1da14a23-ceaa-4c64-9bf8-3486c3d88775 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e03a2be8ac6 namespace: kserve-ci-e2e-test resourceVersion: "22244" uid: 87984995-6169-47a4-acf4-6ffb15407106 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T18:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:40:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:44Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e03bb352980 namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: 0318819d-9319-4f17-a6a5-1a1411d9ce1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e84aea1f6b6 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: b051f1b4-7c5b-42c7-907a-585d04d2a06f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e84aea29d62 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: e8721672-a30e-4604-8bc5-13572eb13b6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: d3410be8-a262-4434-beab-ca64a7d2774f kind: Event lastTimestamp: "2026-07-01T18:40:44Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:44Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj.18be3e852fa83884 namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: 63dfc307-320c-4674-8ff5-c957a39ded39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9 namespace: kserve-ci-e2e-test resourceVersion: "22195" uid: 18d25bd1-28c1-49c2-9b83-81575b39878c kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created pod: error-404-isvc-c1eb4-predictor-5bf4ddd5b9-j8vpj' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor-5bf4ddd5b9.18be3e037ec0b1a8 namespace: kserve-ci-e2e-test resourceVersion: "22203" uid: c4d2af34-2203-4844-bed9-3849c3c28455 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: a346d340-7729-4583-82ba-2e597a6e092c kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Scaled up replica set error-404-isvc-c1eb4-predictor-5bf4ddd5b9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4-predictor.18be3e037ce01c14 namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 99f4ba2f-f09e-4cd4-9023-31afac9c2240 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: a34aad1d-91b5-4a26-a4e1-8df5dd8dd79a kind: Event lastTimestamp: "2026-07-01T18:31:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:57Z" name: error-404-isvc-c1eb4-predictor.18be3e06fb84c032 namespace: kserve-ci-e2e-test resourceVersion: "22505" uid: a295beb9-b21b-45a0-9793-25432a8e38c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: a34aad1d-91b5-4a26-a4e1-8df5dd8dd79a kind: Event lastTimestamp: "2026-07-01T18:31:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:57Z" name: error-404-isvc-c1eb4-predictor.18be3e06fbfb366b namespace: kserve-ci-e2e-test resourceVersion: "22506" uid: f54fbe9e-854d-4d64-8031-a986db2af60a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:32:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22379" uid: a34aad1d-91b5-4a26-a4e1-8df5dd8dd79a kind: Event lastTimestamp: "2026-07-01T18:32:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:57Z" name: error-404-isvc-c1eb4-predictor.18be3e0df98a425a namespace: kserve-ci-e2e-test resourceVersion: "23234" uid: 4c89078a-dd53-4264-be0f-ed040d92103b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:32:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22379" uid: a34aad1d-91b5-4a26-a4e1-8df5dd8dd79a kind: Event lastTimestamp: "2026-07-01T18:32:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:57Z" name: error-404-isvc-c1eb4-predictor.18be3e0df9d53818 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: cf3f9bec-392d-4417-9336-99770f3ff7cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: 2d5b7fd8-b987-4899-b010-5fd363913d28 kind: Event lastTimestamp: "2026-07-01T18:32:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:24Z" name: error-404-isvc-c1eb4.18be3e037cebcf0c namespace: kserve-ci-e2e-test resourceVersion: "22918" uid: 1e50a6bc-b9a9-41c4-b374-3835bc32a89a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: 2d5b7fd8-b987-4899-b010-5fd363913d28 kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-c1eb4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c1eb4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4.18be3e037d8a7c95 namespace: kserve-ci-e2e-test resourceVersion: "22199" uid: eaea6f53-a091-4778-8f5f-b96384d01b7d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: 2d5b7fd8-b987-4899-b010-5fd363913d28 kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c1eb4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:27Z" name: error-404-isvc-c1eb4.18be3e037d8ad3f1 namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 41a1b7b2-4dde-4e8d-886b-c3aaa84ad97e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22916" uid: 2d5b7fd8-b987-4899-b010-5fd363913d28 kind: Event lastTimestamp: "2026-07-01T18:32:24Z" message: InferenceService [error-404-isvc-c1eb4] is Ready metadata: creationTimestamp: "2026-07-01T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:24Z" name: error-404-isvc-c1eb4.18be3e10cb2c0164 namespace: kserve-ci-e2e-test resourceVersion: "22917" uid: fe131299-dba2-4870-8b06-3162d6400c4f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:21:58.007244Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:21:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:21:58Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7ef72ae96a namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: a0d40e01-f679-4b39-88c5-1543b15ef365 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c5c10-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:58Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f01b779ef namespace: kserve-ci-e2e-test resourceVersion: "17628" uid: d6484e48-935f-48eb-aee6-2f22e1ee28c3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f35625076 namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: 04b6935c-1095-4563-8326-50fdd569d3ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f36ad5750 namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: 89ac476f-0e0a-4d89-9658-a8b3438a9e51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f3965b6fb namespace: kserve-ci-e2e-test resourceVersion: "17654" uid: aff1423f-9ddb-4c16-b407-41b68cf8ccb7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f39e19b5a namespace: kserve-ci-e2e-test resourceVersion: "17655" uid: 17ad57fc-0d19-41e4-a00d-7c3679a8d009 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f39ebddcc namespace: kserve-ci-e2e-test resourceVersion: "17656" uid: fa511da3-4d3b-417a-a6f9-dfba05b570d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f3cc33704 namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: f8949b5f-8298-4147-af19-d06ffbf0eb41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f3d358f89 namespace: kserve-ci-e2e-test resourceVersion: "17658" uid: e1899291-d7ca-4f6e-9ad4-b7472ddd9b18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:22:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:16Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3d7f754587a0 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 96fc380e-3b45-4b92-a1ea-6b4e5a67b274 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3e00277afcd5 namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: 937269c6-fc44-403f-a873-2bda00d99648 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:12Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3e00277b9590 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: e9fcd2c8-d993-4047-9812-b75e756e4f75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 6727f26d-7a38-47ac-95b0-74d409638145 kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:16Z" name: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2.18be3e00e9a58968 namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: b64c2247-b6ea-4cf7-8ccd-c721947cb858 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c5c10-predictor-dd55b8f87 namespace: kserve-ci-e2e-test resourceVersion: "17601" uid: aaa3cef4-22b5-490c-be3b-4b993c4c1b81 kind: Event lastTimestamp: "2026-07-01T18:21:58Z" message: 'Created pod: error-404-isvc-c5c10-predictor-dd55b8f87-dncv2' metadata: creationTimestamp: "2026-07-01T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:58Z" name: error-404-isvc-c5c10-predictor-dd55b8f87.18be3d7ef6ceb629 namespace: kserve-ci-e2e-test resourceVersion: "17612" uid: 3bf192d2-1832-42cd-bb4e-871e085433bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: 489d03c2-4ecc-47f7-9dd6-b03b5ed5cfb2 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Scaled up replica set error-404-isvc-c5c10-predictor-dd55b8f87 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:57Z" name: error-404-isvc-c5c10-predictor.18be3d7ef52e66fd namespace: kserve-ci-e2e-test resourceVersion: "17604" uid: 0bd7f7b9-10b0-4655-9e8b-a8616b748302 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17600" uid: 400f8959-a611-471c-8c7f-19b7f781d284 kind: Event lastTimestamp: "2026-07-01T18:22:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:27Z" name: error-404-isvc-c5c10-predictor.18be3d827384d0ee namespace: kserve-ci-e2e-test resourceVersion: "17953" uid: 242b09fc-64c3-4303-ac35-c6fe9d3eb271 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17600" uid: 400f8959-a611-471c-8c7f-19b7f781d284 kind: Event lastTimestamp: "2026-07-01T18:22:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:28Z" name: error-404-isvc-c5c10-predictor.18be3d8273ef1396 namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: c32acf57-76dc-4c2d-84e9-eaa31f3b9bf4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:22:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 400f8959-a611-471c-8c7f-19b7f781d284 kind: Event lastTimestamp: "2026-07-01T18:23:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:13Z" name: error-404-isvc-c5c10-predictor.18be3d89715b0438 namespace: kserve-ci-e2e-test resourceVersion: "18346" uid: cfa01e1e-d4bb-4c0a-a4b9-a295eca968c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:22:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 400f8959-a611-471c-8c7f-19b7f781d284 kind: Event lastTimestamp: "2026-07-01T18:23:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:13Z" name: error-404-isvc-c5c10-predictor.18be3d8971c2a8a0 namespace: kserve-ci-e2e-test resourceVersion: "18347" uid: 0a1864df-8779-4b2d-876c-ab3a53e14449 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: a59424f3-8f84-4fa2-bdbe-490dc418e6a2 kind: Event lastTimestamp: "2026-07-01T18:22:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:46Z" name: error-404-isvc-c5c10.18be3d7ef502810f namespace: kserve-ci-e2e-test resourceVersion: "18110" uid: b72f8f27-1936-4d8a-bd7f-817ebf74a93e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: a59424f3-8f84-4fa2-bdbe-490dc418e6a2 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-c5c10": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c5c10": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:57Z" name: error-404-isvc-c5c10.18be3d7ef5cafcf7 namespace: kserve-ci-e2e-test resourceVersion: "17607" uid: cad4159b-c5c1-4a60-b1c2-f39c3228af44 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: a59424f3-8f84-4fa2-bdbe-490dc418e6a2 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c5c10": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:57Z" name: error-404-isvc-c5c10.18be3d7ef5cb416c namespace: kserve-ci-e2e-test resourceVersion: "17609" uid: b41b04b6-b281-4a1e-b6b0-cee727cd1701 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: a59424f3-8f84-4fa2-bdbe-490dc418e6a2 kind: Event lastTimestamp: "2026-07-01T18:22:46Z" message: InferenceService [error-404-isvc-c5c10] is Ready metadata: creationTimestamp: "2026-07-01T18:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:46Z" name: error-404-isvc-c5c10.18be3d8a306d11e1 namespace: kserve-ci-e2e-test resourceVersion: "18109" uid: 4dbb42e7-7a89-41d1-a5c1-580bfadc5beb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:40:42.005791Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fbcf2-predictor-748c594f6f-tlrls to ip-10-0-132-161.ec2.internal metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:40:42Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84aab3a526 namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 7dadf077-732e-4a55-87e2-c3b68c6fa709 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fbcf2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84b8073d02 namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: b5409ed5-7c91-4b0f-852f-45461d1ab421 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84e832caab namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 1b914234-65f4-43a2-bba6-237426cdf3af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84ea017e56 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 3d1abe95-5bb6-4818-844b-2bad944f52f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84ee29011b namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: b9e317ac-12c3-49e2-8a7b-f992f221595c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84eea30c1e namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 9b7bc520-77bc-4577-a045-1e140a14504e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84eeadbc75 namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 4f91179b-1dd0-46e6-92b7-12e687789fbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84f1988830 namespace: kserve-ci-e2e-test resourceVersion: "26919" uid: 2c959b0b-87fa-4f6c-ba38-231fb5ddbd3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e84f20c6fc5 namespace: kserve-ci-e2e-test resourceVersion: "26920" uid: 8330d1f8-d60f-4fdb-88d6-d84bc33f37b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:41:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:20Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e853135e809 namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: 3c451760-0a44-4a50-a939-3564c3745b9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e950d76870b namespace: kserve-ci-e2e-test resourceVersion: "27778" uid: cfe1a83c-7b4b-44a4-beb3-167714fda1eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e950d77bf99 namespace: kserve-ci-e2e-test resourceVersion: "27779" uid: 871c1540-2db6-4e31-bd54-a28d8f3ba065 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 3f64d70e-353c-4fe5-bdbc-52e32677e226 kind: Event lastTimestamp: "2026-07-01T18:41:55Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:55Z" name: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls.18be3e95b95130fe namespace: kserve-ci-e2e-test resourceVersion: "27842" uid: d059b433-9fa4-4835-89e6-842a27391201 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-161.ec2.internal source: component: kubelet host: ip-10-0-132-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fbcf2-predictor-748c594f6f namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: 6dd458ec-d65b-49d8-929f-ff4ef3f8bc91 kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: 'Created pod: error-404-isvc-fbcf2-predictor-748c594f6f-tlrls' metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:40:41Z" name: error-404-isvc-fbcf2-predictor-748c594f6f.18be3e84aa25881b namespace: kserve-ci-e2e-test resourceVersion: "26865" uid: 4db3c057-ec90-4d17-9fd0-f69cb08015e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: ac273be5-a9a3-4c88-8ef0-a6f6205da2e4 kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: Scaled up replica set error-404-isvc-fbcf2-predictor-748c594f6f from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:40:41Z" name: error-404-isvc-fbcf2-predictor.18be3e84a87d3856 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 5296a554-7c9b-4fc7-889a-3607b442d386 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:40:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: bd353c26-d618-4b23-af7c-68b98e02b679 kind: Event lastTimestamp: "2026-07-01T18:41:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:27Z" name: error-404-isvc-fbcf2-predictor.18be3e88270848bd namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: 82d41d34-6bac-470c-8ae7-da408618e5f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:40:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: bd353c26-d618-4b23-af7c-68b98e02b679 kind: Event lastTimestamp: "2026-07-01T18:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:27Z" name: error-404-isvc-fbcf2-predictor.18be3e882756f25d namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 3eb60f2f-6fab-4284-9f73-043fb514a59a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "27048" uid: bd353c26-d618-4b23-af7c-68b98e02b679 kind: Event lastTimestamp: "2026-07-01T18:41:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:42Z" name: error-404-isvc-fbcf2-predictor.18be3e92a3b23677 namespace: kserve-ci-e2e-test resourceVersion: "27567" uid: 6d843ad3-341c-44f0-ac10-ee37a6d88331 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "27048" uid: bd353c26-d618-4b23-af7c-68b98e02b679 kind: Event lastTimestamp: "2026-07-01T18:41:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:42Z" name: error-404-isvc-fbcf2-predictor.18be3e92a42e629a namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 76d32633-b3e2-40d0-bad9-d020fc7946e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: b11b3c07-1f92-4b4b-8f68-92a0bbd07e5a kind: Event lastTimestamp: "2026-07-01T18:41:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:30Z" name: error-404-isvc-fbcf2.18be3e84a8776cdf namespace: kserve-ci-e2e-test resourceVersion: "27315" uid: 9f3d3060-e207-4149-9a0e-e17a3bb12cf9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: b11b3c07-1f92-4b4b-8f68-92a0bbd07e5a kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-fbcf2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbcf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:40:41Z" name: error-404-isvc-fbcf2.18be3e84a8efb84b namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 5f8ef1a5-d318-4a52-aed3-0f236fb50861 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: b11b3c07-1f92-4b4b-8f68-92a0bbd07e5a kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbcf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:40:41Z" name: error-404-isvc-fbcf2.18be3e84a8eff802 namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: d455003d-984c-4f8b-a50d-416d494f9e13 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: b11b3c07-1f92-4b4b-8f68-92a0bbd07e5a kind: Event lastTimestamp: "2026-07-01T18:41:30Z" message: InferenceService [error-404-isvc-fbcf2] is Ready metadata: creationTimestamp: "2026-07-01T18:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:30Z" name: error-404-isvc-fbcf2.18be3e8fec4aab08 namespace: kserve-ci-e2e-test resourceVersion: "27314" uid: 579aebbc-a077-4a5c-94bf-4af7f1688f8d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:19:27.853321Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q to ip-10-0-136-152.ec2.internal metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:19:27Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5c014be2bc namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 3dfde938-970e-4783-930b-70ab893862cf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5c121cada3 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 449e22b2-c486-4a8a-bb42-cb7739469db2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5c3ea31025 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 80d2ab8d-0281-4f24-898e-7bea6f6fb287 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5c40345a36 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 83610b8c-6638-457c-8a64-2017779045dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5c43cf0e03 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: 5a076ebc-5f20-4947-ab63-80c2d693cb18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5c4441deed namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: c37dad4a-7acc-45c0-bc62-d1995d0e16ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:19:33Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-01T18:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:33Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d5d42068b4d namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 456fb0f2-7a7d-463d-ab81-f9eb74a7eec8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 27.685s (27.685s including waiting). Image size: 1559186858 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:00Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d63b4357093 namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 75138028-1b92-45f3-87a4-37197afdf790 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d63ba3a7948 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 06e76389-92fd-4a34-ad29-ecf658dc8183 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d63bb8910f7 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: b47945fa-f9a2-4457-95ad-2866df293915 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d63bb96724a namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 82476fc9-119b-4e6f-b705-ca48ae493343 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.65s (3.65s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d64953456e0 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: d140ea2d-8033-4741-9581-4789d61a79d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d649ae359b7 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 597e6374-2765-49a4-be72-273201f2cf73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d649bcc8747 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 74860598-ec56-4c33-b250-881c54a09796 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:20:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:22:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:02Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d64ff0aa027 namespace: kserve-ci-e2e-test resourceVersion: "17732" uid: e0741978-b5d3-4e3d-8e98-fc6ea6207ad6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d7ef327ee83 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: 678c4828-055c-4526-b186-110151ecdcb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d7ef32820e9 namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: 80701c02-b584-4b45-b192-81638cddf098 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 58761e7f-5592-4b8d-a9c8-876f55a83277 kind: Event lastTimestamp: "2026-07-01T18:22:02Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:02Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q.18be3d80016a8483 namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: 5a1b4995-1a82-42bd-98c0-8591f444c5fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7c964dbd44 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 122d2e71-ea43-4ca8-8752-b94b73c67176 kind: Event lastTimestamp: "2026-07-01T18:19:27Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7c964dbd44-9zh7q' metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:27Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44.18be3d5c00d9cb14 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 88a41712-0655-49ba-a647-5411fe497c98 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 17cba480-fbe2-4ea3-91de-1c923b9ef8e6 kind: Event lastTimestamp: "2026-07-01T18:19:27Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7c964dbd44 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:27Z" name: isvc-sklearn-graph-1-predictor.18be3d5bfdcc04da namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 96b229e0-2434-4ad6-b56b-42fc64e24c23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 8b419f72-0ec0-4994-a7a3-6fe158138784 kind: Event lastTimestamp: "2026-07-01T18:20:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:12Z" name: isvc-sklearn-graph-1-predictor.18be3d5f7c681517 namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: 954d8488-f1bf-4c5c-8896-47816b93bc4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 8b419f72-0ec0-4994-a7a3-6fe158138784 kind: Event lastTimestamp: "2026-07-01T18:20:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:12Z" name: isvc-sklearn-graph-1-predictor.18be3d5f7cd724ce namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: 5f8ce92e-6d39-4c4b-961a-7b0c50778a27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:20:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 8b419f72-0ec0-4994-a7a3-6fe158138784 kind: Event lastTimestamp: "2026-07-01T18:21:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:42Z" name: isvc-sklearn-graph-1-predictor.18be3d69f9389185 namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: 51863913-930e-4f25-823f-7ccb57f2c1ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:20:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 8b419f72-0ec0-4994-a7a3-6fe158138784 kind: Event lastTimestamp: "2026-07-01T18:21:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:42Z" name: isvc-sklearn-graph-1-predictor.18be3d69f9d2eb24 namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: 7917864e-4895-463e-8fd0-858beadc5b0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 3f15683e-c705-4401-a84b-fa59a193d151 kind: Event lastTimestamp: "2026-07-01T18:21:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:22Z" name: isvc-sklearn-graph-1.18be3d5bfdcc8d89 namespace: kserve-ci-e2e-test resourceVersion: "17171" uid: 712a105a-7c02-4303-946a-c09059dfe156 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: 3f15683e-c705-4401-a84b-fa59a193d151 kind: Event lastTimestamp: "2026-07-01T18:21:22Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T18:21:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:22Z" name: isvc-sklearn-graph-1.18be3d76b70e03ac namespace: kserve-ci-e2e-test resourceVersion: "17170" uid: 1cdc9e7d-e264-4a54-9ed4-4d6608c7f2ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:19:28.220760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-84b568d556-vxgvf to ip-10-0-136-152.ec2.internal metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d5c17328df8 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 74f74e68-1074-4bb6-8eca-63f7f5bd5424 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d5c30ae3346 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: d523f333-9bb1-409a-948f-9bd2553458f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d5c3240b1fc namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 97869955-2779-4dab-82ea-157ada53a44c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d5c34d8a595 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 7b00117d-cd84-4af3-a259-6114acce1d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d5c3568728c namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 5c75631b-334a-4c78-bc95-35515ff6b07b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:19:33Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-01T18:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:33Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d5d421c6ebb namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: dd9348d3-7fff-4ce3-8a18-19d2614a5b43 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 27.326s (27.326s including waiting). Image size: 1559186858 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:00Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d639eed1284 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 1d0f04b2-fc76-4fcf-adc5-2d5c8862cb38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:00Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d63a3b6a06c namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: bcf645fa-3cb1-4b41-98ef-b36487b0fbbc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:00Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d63a4eb3b41 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: 867ae0d6-8188-4132-b78c-4b4160072851 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:00Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:00Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d63a4fa481a namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: fa59c4ec-07cf-4a03-8ac3-8039d1626ba9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.101s (4.101s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d649971a37d namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: 59ca6b23-93d0-4a27-8bb7-269466899e77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d649d59d916 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: bbfcaf4d-f1b7-442f-989a-40ce279eaafa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d649e40a274 namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 979b383d-0e75-4cdd-bd51-8446d4ad4d57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:20:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:22:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:01Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d64c30a3b81 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: dfb3fc11-c3a2-4ebe-94a2-43477b17f35d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d7eefe4c9ee namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 2fec7427-80f2-4b92-bff4-f49ff1e1945f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d7eefe5d24c namespace: kserve-ci-e2e-test resourceVersion: "17563" uid: d714ad4e-1a2f-41f8-9b66-1a1f93941dc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: b070bf96-654c-4b52-95dd-df1b1be88642 kind: Event lastTimestamp: "2026-07-01T18:22:01Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:01Z" name: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf.18be3d7fc5925013 namespace: kserve-ci-e2e-test resourceVersion: "17686" uid: 8fc5b62e-868b-4762-9d51-e82510a83cb9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-84b568d556 namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 0a50c3db-40f8-4189-96e7-98e39f598c52 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-84b568d556-vxgvf' metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor-84b568d556.18be3d5c16b564c5 namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: d749ba51-d8f3-4325-a687-7a5a2929ca5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: f8f7af01-47c7-43af-af66-bb704311c4d5 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-84b568d556 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:28Z" name: isvc-sklearn-graph-2-predictor.18be3d5c1120a03b namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 91d2afbe-e447-44e1-a224-eff88458e411 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: a8e1e851-21a7-4a1e-893c-557af331a914 kind: Event lastTimestamp: "2026-07-01T18:20:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:28Z" name: isvc-sklearn-graph-2-predictor.18be3d5f8f9d7fb3 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: 0b6af861-8244-443b-bb2b-ab9079b6c6a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: a8e1e851-21a7-4a1e-893c-557af331a914 kind: Event lastTimestamp: "2026-07-01T18:20:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:28Z" name: isvc-sklearn-graph-2-predictor.18be3d5f900bed43 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: 50157618-8ecd-4038-883c-952abfc9bf41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:20:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: a8e1e851-21a7-4a1e-893c-557af331a914 kind: Event lastTimestamp: "2026-07-01T18:21:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:43Z" name: isvc-sklearn-graph-2-predictor.18be3d6d8b562a04 namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 9658b378-6eae-480a-b0a5-c9b1d132a96a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:20:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: a8e1e851-21a7-4a1e-893c-557af331a914 kind: Event lastTimestamp: "2026-07-01T18:21:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:43Z" name: isvc-sklearn-graph-2-predictor.18be3d6d8bc08caa namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: 8b007b73-2cfe-4d14-a34a-080bd650db3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: 26caeacd-12ef-41a3-a402-a8551cc21d9d kind: Event lastTimestamp: "2026-07-01T18:21:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:21Z" name: isvc-sklearn-graph-2.18be3d5c114a6dc8 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 2a2a043c-64b7-40bd-a9de-e1bb96f31de3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 26caeacd-12ef-41a3-a402-a8551cc21d9d kind: Event lastTimestamp: "2026-07-01T18:21:21Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T18:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:21Z" name: isvc-sklearn-graph-2.18be3d767b35a86f namespace: kserve-ci-e2e-test resourceVersion: "17154" uid: b56a23df-99a8-4044-88d9-a5e1e3af297e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:19:28.066081Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-hcw7d to ip-10-0-136-152.ec2.internal metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:19:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5c0dfa47bc namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 61fa5e7f-c693-44ad-8cfd-8f6492ae7b73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5c181e558d namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: d7fe3b28-93f8-468b-af7d-ffe97294cab1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:19:29Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:19:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:19:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5c4bd027b6 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: e93e500b-d953-4e56-891f-8013394bece7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:19:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5c4d5367d4 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 928544ed-9183-4f55-9339-d6467d9c287f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:19:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:19:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5c513f7c21 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 38f09d87-a424-438a-8037-94b686ab3d0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:19:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:19:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5c51bb2cd8 namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 8b473be2-8579-429c-aeb1-2699a250db7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:19:33Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T18:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d5d423617f8 namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 8c75a5a6-8b8d-45b2-a9ee-b3f26a635330 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.994s (27.994s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d63c6d0bced namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 9136ed58-d74c-44d4-bcd1-5c3ade1d03f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d63cb0bef92 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 62e00351-940b-4e4c-8fe4-8e43e27a958b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d63cb6f1ca5 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 1cb6c917-32c2-478c-94ad-f78a57c84368 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d63cb7baec5 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: ec48b383-d967-4645-ac7d-22e25bc5dcd3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.394s (3.394s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d6495ce292a namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 151f6a27-f543-48ce-a45b-e1e7b31a9588 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d649a5cd6e8 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: a19fb68b-13f6-4f26-9616-f0ffdb444f5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:20:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d649bbbbf2e namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 72f3a68b-646f-4c1e-ad63-d830bdafca9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:20:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:22:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d64c32575d9 namespace: kserve-ci-e2e-test resourceVersion: "17689" uid: c652bccb-23e5-4f28-b222-f780a4a58224 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d7eecae6627 namespace: kserve-ci-e2e-test resourceVersion: "17532" uid: 5b705278-7b25-4e3d-8116-88f472c6631f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d7eecaee39f namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 17d3ea83-b147-47e2-a893-33e6394e6347 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 20a33dae-136c-4a92-8a1c-e51e0ca7a640 kind: Event lastTimestamp: "2026-07-01T18:22:01Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-hcw7d.18be3d7fc5936f9b namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: d318a7b5-a4fc-4530-847d-fa972ec688c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-152.ec2.internal source: component: kubelet host: ip-10-0-136-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: c804bb6a-a66b-4e55-8936-f992f5b75f5e kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-hcw7d' metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:28Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be3d5c0d77c044 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 07458251-d4ca-4644-8239-bd8a5717be7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 7e1063b5-ebd9-41ba-ac1e-1af2439a986c kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:28Z" name: isvc-xgboost-graph-predictor.18be3d5c0a61843d namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: c34e2c83-b791-449c-a47c-44e7ffadcfc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:19:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 342ca3d2-ff96-4ad7-8865-34bb93c583ae kind: Event lastTimestamp: "2026-07-01T18:20:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:43Z" name: isvc-xgboost-graph-predictor.18be3d5f88fab8e2 namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: b93a5cf0-85c4-4f0e-888a-bb21101d1b54 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:19:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 342ca3d2-ff96-4ad7-8865-34bb93c583ae kind: Event lastTimestamp: "2026-07-01T18:20:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:43Z" name: isvc-xgboost-graph-predictor.18be3d5f8947e66b namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: a11e64fd-bf29-4677-b8bc-c6b02b06e667 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 342ca3d2-ff96-4ad7-8865-34bb93c583ae kind: Event lastTimestamp: "2026-07-01T18:21:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:28Z" name: isvc-xgboost-graph-predictor.18be3d7103533464 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 3942d654-1128-46c6-a404-9ca71a326c7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 342ca3d2-ff96-4ad7-8865-34bb93c583ae kind: Event lastTimestamp: "2026-07-01T18:21:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:28Z" name: isvc-xgboost-graph-predictor.18be3d7103a5298c namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 1b677be8-5055-44b2-985a-6e6178eba691 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: c6f1e937-3f87-4411-8e81-ef9f42c11234 kind: Event lastTimestamp: "2026-07-01T18:21:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:11Z" name: isvc-xgboost-graph.18be3d5c0a6f394f namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: cce7c31d-eb9a-4b8b-a84e-86339e98f277 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: c6f1e937-3f87-4411-8e81-ef9f42c11234 kind: Event lastTimestamp: "2026-07-01T18:21:11Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T18:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:11Z" name: isvc-xgboost-graph.18be3d7426b5721f namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 85b9889c-d766-4114-800a-f9ed996cabf5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:21:47.637024Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17377" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-b57d5bf9d-6cg52 to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:21:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:21:47Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7c8d0ddb5c namespace: kserve-ci-e2e-test resourceVersion: "17382" uid: 00d1eb60-5136-46fc-bf2d-4b1f1f6400ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:48Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: "2026-07-01T18:21:48Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:21:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:21:48Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7ca6ebf810 namespace: kserve-ci-e2e-test resourceVersion: "17397" uid: 41a2effa-dd49-4636-b667-023abbaf57e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: "2026-07-01T18:21:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:48Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7ca8264b98 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 475e6352-e321-42a8-96cf-841e97b58d12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: "2026-07-01T18:21:48Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T18:21:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:48Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7cac057a28 namespace: kserve-ci-e2e-test resourceVersion: "17401" uid: 7be9c837-7597-422b-896b-03150eea7453 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: "2026-07-01T18:21:48Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T18:21:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:48Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7cac675ac9 namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: 5f2f7861-1541-43f0-b3ec-ed5a649d6685 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:57Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7ee7795b79 namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: a88376a0-8de1-4606-b110-732e8649a655 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b57d5bf9d-6cg52 namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: c2e0f452-f799-4720-b9ef-7969cedb6e2e kind: Event lastTimestamp: "2026-07-01T18:22:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:24Z" name: model-chainer-b57d5bf9d-6cg52.18be3d7f44746b68 namespace: kserve-ci-e2e-test resourceVersion: "17925" uid: a06315f3-efcd-4c87-bea2-01247b6069bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-b57d5bf9d namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: f865cf86-f0a1-43a6-bd4e-758c271be459 kind: Event lastTimestamp: "2026-07-01T18:21:47Z" message: 'Created pod: model-chainer-b57d5bf9d-6cg52' metadata: creationTimestamp: "2026-07-01T18:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:47Z" name: model-chainer-b57d5bf9d.18be3d7c8c995c16 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: e4b2ee73-5fef-4d5e-91ff-7b35d91a981c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17372" uid: 50684925-021b-45df-8ec2-11b2755fd6c1 kind: Event lastTimestamp: "2026-07-01T18:21:47Z" message: Scaled up replica set model-chainer-b57d5bf9d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:47Z" name: model-chainer.18be3d7c8bc90956 namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: b9c71dab-ab42-46c3-9976-8a986e6bfe99 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: 3b4f81eb-bd84-4687-80ed-243133d7dbe5 kind: Event lastTimestamp: "2026-07-01T18:21:47Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T18:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:47Z" name: model-chainer.18be3d7c8f39d251 namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: d7022945-5c3f-4a79-b950-2a9e416d7971 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:33:23.042618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23447" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-293b3-8f6998d6c-5gtvf to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e1e76752c13 namespace: kserve-ci-e2e-test resourceVersion: "23452" uid: 418c8cba-50e1-4109-a496-8f691ed85ce8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-293b3-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e1e8126724e namespace: kserve-ci-e2e-test resourceVersion: "23469" uid: 447a75e8-e825-44bc-92f3-37c11ad9e40e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:24Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23456" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:33:24Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:33:24Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e1eb3ae3b20 namespace: kserve-ci-e2e-test resourceVersion: "23477" uid: f1fd90d4-d764-4553-be25-38057cb96ab7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-293b3} kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:33:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:24Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e1eb4f67c82 namespace: kserve-ci-e2e-test resourceVersion: "23479" uid: 9a1ec1b7-8629-4ddd-a2c0-849c0cc828d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-293b3} kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:33:24Z" message: 'Created container: sequence-graph-293b3' metadata: creationTimestamp: "2026-07-01T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:24Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e1eb8651187 namespace: kserve-ci-e2e-test resourceVersion: "23480" uid: dbbc6820-90b0-4ab5-8bc9-268751256e29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-293b3} kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:33:24Z" message: Started container sequence-graph-293b3 metadata: creationTimestamp: "2026-07-01T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:33:24Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e1eb8ce0801 namespace: kserve-ci-e2e-test resourceVersion: "23481" uid: 4415e521-4abf-4001-8722-615f250a1e3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-293b3} kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Stopping container sequence-graph-293b3 metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:37Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e91a59724dc namespace: kserve-ci-e2e-test resourceVersion: "27375" uid: 04d9789c-a54e-48f1-9253-a6cf26c21ec3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-293b3} kind: Pod name: sequence-graph-293b3-8f6998d6c-5gtvf namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 3191b6b2-9429-4e38-80e3-351efe491d9e kind: Event lastTimestamp: "2026-07-01T18:42:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:05Z" name: sequence-graph-293b3-8f6998d6c-5gtvf.18be3e924572f9f2 namespace: kserve-ci-e2e-test resourceVersion: "27924" uid: ba4e8a4f-6d29-47e6-98d1-956a8771ba72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-293b3-8f6998d6c namespace: kserve-ci-e2e-test resourceVersion: "23442" uid: 312bb74c-ff9f-4d48-8202-c58d90196218 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: 'Created pod: sequence-graph-293b3-8f6998d6c-5gtvf' metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3-8f6998d6c.18be3e1e76033e0c namespace: kserve-ci-e2e-test resourceVersion: "23448" uid: a63645ea-81ca-4740-b03a-a271cb2eb9f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23440" uid: 87b79447-718e-4564-b5f0-1967b03aaa13 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: Scaled up replica set sequence-graph-293b3-8f6998d6c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3.18be3e1e751fa91f namespace: kserve-ci-e2e-test resourceVersion: "23444" uid: 031f58d6-a0d7-4ae5-b564-ee5e081058ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23462" uid: 657f6dbb-3fea-4ca6-a68a-8683b960c537 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: InferenceGraph [sequence-graph-293b3] is Ready metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3.18be3e1e7b13695f namespace: kserve-ci-e2e-test resourceVersion: "23465" uid: 32799621-1cd2-4b74-bdff-6e458b648907 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23435" uid: 657f6dbb-3fea-4ca6-a68a-8683b960c537 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: 'Failed to update status for InferenceGraph "sequence-graph-293b3": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-293b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3.18be3e1e7dae1caf namespace: kserve-ci-e2e-test resourceVersion: "23466" uid: 613fe3bb-ce89-4c0b-afdf-f6e77e1ad7d2 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23435" uid: 657f6dbb-3fea-4ca6-a68a-8683b960c537 kind: Event lastTimestamp: "2026-07-01T18:33:23Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-293b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:23Z" name: sequence-graph-293b3.18be3e1e7dae692f namespace: kserve-ci-e2e-test resourceVersion: "23467" uid: c75e86c8-b98d-4c04-858c-be9bfe146b6e reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:33:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23441" uid: ebb74fa9-1049-447a-97df-622be1d9d0c6 kind: Event lastTimestamp: "2026-07-01T18:34:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:34:23Z" name: sequence-graph-293b3.18be3e21f4235169 namespace: kserve-ci-e2e-test resourceVersion: "23905" uid: 3d8dd6dd-a968-4c88-a8ec-43a27cb7c7cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:33:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23441" uid: ebb74fa9-1049-447a-97df-622be1d9d0c6 kind: Event lastTimestamp: "2026-07-01T18:34:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:34:23Z" name: sequence-graph-293b3.18be3e21f47d1a05 namespace: kserve-ci-e2e-test resourceVersion: "23906" uid: ed2b8e7d-a4a8-4831-9a19-de116790699d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:32:12.751569Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22652" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9b590-78448686f8-prj8j to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:32:12Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e0e18c8ed73 namespace: kserve-ci-e2e-test resourceVersion: "22658" uid: 0c4f43df-f40f-4838-86a7-b3cc2a45ae2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-9b590-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:12Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e0e25933434 namespace: kserve-ci-e2e-test resourceVersion: "22673" uid: faf66b32-1944-4fa1-babd-231def869799 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:13Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:32:13Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e0e558b4672 namespace: kserve-ci-e2e-test resourceVersion: "22680" uid: a11e90e5-092e-4d1d-a110-7fbf638aa91a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9b590} kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:13Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e0e56bcd818 namespace: kserve-ci-e2e-test resourceVersion: "22682" uid: 7a90dc69-9458-48d7-9b88-a3640af62bdd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9b590} kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:13Z" message: 'Created container: sequence-graph-9b590' metadata: creationTimestamp: "2026-07-01T18:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:13Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e0e59ae21f9 namespace: kserve-ci-e2e-test resourceVersion: "22683" uid: d63d3872-ef29-4015-a799-3ec557d5c3dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9b590} kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:13Z" message: Started container sequence-graph-9b590 metadata: creationTimestamp: "2026-07-01T18:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:13Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e0e5a192820 namespace: kserve-ci-e2e-test resourceVersion: "22684" uid: cb7f8988-c2ad-4eb7-bd7a-df4e41ea931d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9b590} kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: Stopping container sequence-graph-9b590 metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:22Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e10716aa7a1 namespace: kserve-ci-e2e-test resourceVersion: "22782" uid: 854407e7-090c-42c9-8b50-b79956a6db1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9b590} kind: Pod name: sequence-graph-9b590-78448686f8-prj8j namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 361eeffa-efd3-4930-a1c5-95ffb0f1f948 kind: Event lastTimestamp: "2026-07-01T18:32:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:50Z" name: sequence-graph-9b590-78448686f8-prj8j.18be3e10f8d15098 namespace: kserve-ci-e2e-test resourceVersion: "23174" uid: c9bb454c-9814-4747-82e8-56bd1405495a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9b590-78448686f8 namespace: kserve-ci-e2e-test resourceVersion: "22649" uid: 9ff3186f-f55d-4a76-88e4-f3f33a978023 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: 'Created pod: sequence-graph-9b590-78448686f8-prj8j' metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:12Z" name: sequence-graph-9b590-78448686f8.18be3e0e1865b988 namespace: kserve-ci-e2e-test resourceVersion: "22654" uid: ea92a0d8-1f26-4f82-81f1-d1919105190a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9b590 namespace: kserve-ci-e2e-test resourceVersion: "22647" uid: 4481ebe2-17ba-4562-b012-064aeb2c2c26 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: Scaled up replica set sequence-graph-9b590-78448686f8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:12Z" name: sequence-graph-9b590.18be3e0e177cbeb1 namespace: kserve-ci-e2e-test resourceVersion: "22650" uid: 8e492be4-f396-47f8-aafe-350f296419b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9b590 namespace: kserve-ci-e2e-test resourceVersion: "22667" uid: ee339b36-90ab-489e-b891-4debc1bef84c kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: InferenceGraph [sequence-graph-9b590] is Ready metadata: creationTimestamp: "2026-07-01T18:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:12Z" name: sequence-graph-9b590.18be3e0e1cfa298c namespace: kserve-ci-e2e-test resourceVersion: "22669" uid: 58b7c78e-7925-47b6-a9e6-2665591cca1f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:22:57.928036Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18199" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c5c10-7b978df757-rw5r6 to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:22:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:22:57Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3d8ceab999a1 namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: 7ebd7c3a-14dd-436e-99dd-a2bb5ee95db6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:58Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18210" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: "2026-07-01T18:22:58Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:22:58Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3d8d041f5ea5 namespace: kserve-ci-e2e-test resourceVersion: "18223" uid: c20f9e18-bcdd-44d2-a71f-e42c0ccf08f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5c10} kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: "2026-07-01T18:22:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:58Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3d8d0543d1d7 namespace: kserve-ci-e2e-test resourceVersion: "18226" uid: 2c4ed71b-3773-4a6b-b6e4-ad2b7818eb5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5c10} kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: "2026-07-01T18:22:58Z" message: 'Created container: sequence-graph-c5c10' metadata: creationTimestamp: "2026-07-01T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:58Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3d8d07dac259 namespace: kserve-ci-e2e-test resourceVersion: "18227" uid: 8d4c47c9-d069-45eb-b8a1-8c1ed7aed7b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5c10} kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: "2026-07-01T18:22:58Z" message: Started container sequence-graph-c5c10 metadata: creationTimestamp: "2026-07-01T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:58Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3d8d083e7232 namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: 53841318-a719-436a-a36a-5e27f24f1d5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5c10} kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Stopping container sequence-graph-c5c10 metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:12Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3e0016b14288 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: d2891e0d-4ceb-401c-9fc7-a3a006de4861 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c5c10} kind: Pod name: sequence-graph-c5c10-7b978df757-rw5r6 namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: 1901c013-03ed-4930-adf2-b88b17961d79 kind: Event lastTimestamp: "2026-07-01T18:31:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:39Z" name: sequence-graph-c5c10-7b978df757-rw5r6.18be3e0090fa4ff4 namespace: kserve-ci-e2e-test resourceVersion: "22353" uid: e950776c-b2f5-406b-ab11-8e761f2c09f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c5c10-7b978df757 namespace: kserve-ci-e2e-test resourceVersion: "18196" uid: a048bf0a-f3f7-458f-a5b4-87cb9ea3f91c kind: Event lastTimestamp: "2026-07-01T18:22:57Z" message: 'Created pod: sequence-graph-c5c10-7b978df757-rw5r6' metadata: creationTimestamp: "2026-07-01T18:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:57Z" name: sequence-graph-c5c10-7b978df757.18be3d8cea6a938d namespace: kserve-ci-e2e-test resourceVersion: "18201" uid: 7ed41bcc-ca6f-4c58-ba6d-e1c3592313e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18194" uid: c09fec13-2864-4200-8129-c05050b1e8e3 kind: Event lastTimestamp: "2026-07-01T18:22:57Z" message: Scaled up replica set sequence-graph-c5c10-7b978df757 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:57Z" name: sequence-graph-c5c10.18be3d8ce97a8c7b namespace: kserve-ci-e2e-test resourceVersion: "18197" uid: eb5ae2c0-aa9a-4a10-8fb4-4f2a1664e7b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18215" uid: 6bb628b4-4cc9-4d1a-995d-055408c49455 kind: Event lastTimestamp: "2026-07-01T18:22:57Z" message: InferenceGraph [sequence-graph-c5c10] is Ready metadata: creationTimestamp: "2026-07-01T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:58Z" name: sequence-graph-c5c10.18be3d8ceefd0b57 namespace: kserve-ci-e2e-test resourceVersion: "18217" uid: df206899-c859-4068-bc8e-08746b9e5515 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18195" uid: 9d871e0e-f62d-45f0-aebe-8fbb73de7bee kind: Event lastTimestamp: "2026-07-01T18:23:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:12Z" name: sequence-graph-c5c10.18be3d90689d1a70 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 1709382c-9411-44c5-8f3a-53fcebb448b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18195" uid: 9d871e0e-f62d-45f0-aebe-8fbb73de7bee kind: Event lastTimestamp: "2026-07-01T18:23:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:12Z" name: sequence-graph-c5c10.18be3d9068f1be06 namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: ccda2c80-b2d5-4663-becc-0e6254f91fd7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 9d871e0e-f62d-45f0-aebe-8fbb73de7bee kind: Event lastTimestamp: "2026-07-01T18:23:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:27Z" name: sequence-graph-c5c10.18be3d93e7b6391c namespace: kserve-ci-e2e-test resourceVersion: "18439" uid: f2062908-38d7-444b-a062-c058576c1b01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 9d871e0e-f62d-45f0-aebe-8fbb73de7bee kind: Event lastTimestamp: "2026-07-01T18:23:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:27Z" name: sequence-graph-c5c10.18be3d93e81f9019 namespace: kserve-ci-e2e-test resourceVersion: "18440" uid: 1ee0614f-83fc-41f9-a7cd-4720d6559e31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:42:52.267935Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28387" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-1f8d1-76dfc65fbc-hs22t to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:42:52Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3ea2feee3ba2 namespace: kserve-ci-e2e-test resourceVersion: "28393" uid: e06c6e82-df79-444f-9b45-7ebdbb2251ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28389" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:42:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-1f8d1-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:52Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3ea30b234e24 namespace: kserve-ci-e2e-test resourceVersion: "28406" uid: db5b5934-0bd6-4e56-82e8-677cf35406a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28397" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:42:53Z" message: Add eth0 [10.134.0.56/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:42:53Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3ea33bb5e6cf namespace: kserve-ci-e2e-test resourceVersion: "28421" uid: fb830d60-48f7-4b72-ad5f-07e79a8d5066 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1f8d1} kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28389" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:42:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:53Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3ea33cf44ae3 namespace: kserve-ci-e2e-test resourceVersion: "28423" uid: d4f2b2b2-c3f9-41a7-883c-8d7734e0008d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1f8d1} kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28389" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:42:53Z" message: 'Created container: splitter-graph-1f8d1' metadata: creationTimestamp: "2026-07-01T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:53Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3ea34011dee1 namespace: kserve-ci-e2e-test resourceVersion: "28425" uid: 89aa48ec-785b-427c-ae66-8ab90303130c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1f8d1} kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28389" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:42:53Z" message: Started container splitter-graph-1f8d1 metadata: creationTimestamp: "2026-07-01T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:53Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3ea3407931bf namespace: kserve-ci-e2e-test resourceVersion: "28426" uid: 3f030576-2f04-423f-9d2a-2cc371e6e550 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1f8d1} kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28389" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:51:07Z" message: Stopping container splitter-graph-1f8d1 metadata: creationTimestamp: "2026-07-01T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:51:07Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3f1631de451a namespace: kserve-ci-e2e-test resourceVersion: "31873" uid: aeae17e3-beac-46e1-852b-cdf5611b0e7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:51:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1f8d1} kind: Pod name: splitter-graph-1f8d1-76dfc65fbc-hs22t namespace: kserve-ci-e2e-test resourceVersion: "28389" uid: 08cb8d05-b37a-45df-bcc0-614a25013f98 kind: Event lastTimestamp: "2026-07-01T18:51:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:51:34Z" name: splitter-graph-1f8d1-76dfc65fbc-hs22t.18be3f16dcd10766 namespace: kserve-ci-e2e-test resourceVersion: "32111" uid: 9c9016a7-6568-48fa-a536-e9720c687e54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-1f8d1-76dfc65fbc namespace: kserve-ci-e2e-test resourceVersion: "28382" uid: 4cd038c5-f7cf-4521-a2ec-f14218d420d4 kind: Event lastTimestamp: "2026-07-01T18:42:52Z" message: 'Created pod: splitter-graph-1f8d1-76dfc65fbc-hs22t' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:52Z" name: splitter-graph-1f8d1-76dfc65fbc.18be3ea2fe8e7153 namespace: kserve-ci-e2e-test resourceVersion: "28388" uid: ca259005-cba6-4311-a9ba-0010e7a25238 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "28378" uid: f2a15848-1656-4909-912e-e6cacbe201dd kind: Event lastTimestamp: "2026-07-01T18:42:52Z" message: Scaled up replica set splitter-graph-1f8d1-76dfc65fbc from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:52Z" name: splitter-graph-1f8d1.18be3ea2fdc797dd namespace: kserve-ci-e2e-test resourceVersion: "28384" uid: 62a3d24f-c995-4010-a95d-633f7f800321 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "28399" uid: 92ca038e-cf2e-42f7-ba00-bab73fc0ccff kind: Event lastTimestamp: "2026-07-01T18:42:52Z" message: InferenceGraph [splitter-graph-1f8d1] is Ready metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:52Z" name: splitter-graph-1f8d1.18be3ea3036aad4c namespace: kserve-ci-e2e-test resourceVersion: "28404" uid: 6331dca8-7b75-47d8-b3cf-09f68348bfd1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:43:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "28381" uid: 66688776-0d05-4c1b-845a-8c52e5074948 kind: Event lastTimestamp: "2026-07-01T18:43:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:43:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:37Z" name: splitter-graph-1f8d1.18be3ea67cd3a944 namespace: kserve-ci-e2e-test resourceVersion: "28756" uid: 9c87533e-5b93-4c2f-a75e-227b9baf9cf8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:43:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "28381" uid: 66688776-0d05-4c1b-845a-8c52e5074948 kind: Event lastTimestamp: "2026-07-01T18:43:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:43:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:37Z" name: splitter-graph-1f8d1.18be3ea67d3b537f namespace: kserve-ci-e2e-test resourceVersion: "28757" uid: 322f477c-09bf-4f9d-9e77-11122ea69964 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:41:41.955618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27556" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-fbcf2-865d699f8c-46bjm to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:41:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:41:41Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e929ffd6894 namespace: kserve-ci-e2e-test resourceVersion: "27562" uid: b471d127-82c2-42ee-af43-3cfe46784c5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:41:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-fbcf2-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:42Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e92abfefb91 namespace: kserve-ci-e2e-test resourceVersion: "27575" uid: 5afb48d7-54a4-4360-b6e7-19e3d26a5b11 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27565" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:41:42Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:41:42Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e92dd388019 namespace: kserve-ci-e2e-test resourceVersion: "27586" uid: 9ce2a178-eefe-4df9-bec0-623dcfd04cae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fbcf2} kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:41:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:43Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e92de83e9be namespace: kserve-ci-e2e-test resourceVersion: "27588" uid: b6a7a1f3-2aa3-4299-a9b8-30a1afb8e1e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fbcf2} kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:41:43Z" message: 'Created container: splitter-graph-fbcf2' metadata: creationTimestamp: "2026-07-01T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:43Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e92e2487f53 namespace: kserve-ci-e2e-test resourceVersion: "27590" uid: c2808aa9-bf79-4c8d-b2ee-267a555910b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fbcf2} kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:41:43Z" message: Started container splitter-graph-fbcf2 metadata: creationTimestamp: "2026-07-01T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:43Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e92e2b0a2fe namespace: kserve-ci-e2e-test resourceVersion: "27591" uid: 29e8cd39-d9da-4906-9700-7e6114ddbba8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fbcf2} kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Stopping container splitter-graph-fbcf2 metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e94fa240d14 namespace: kserve-ci-e2e-test resourceVersion: "27687" uid: ca170e34-155a-40e9-b68d-33de906d23b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fbcf2} kind: Pod name: splitter-graph-fbcf2-865d699f8c-46bjm namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: 5acf7865-4f8a-48e8-a923-e75376e6e133 kind: Event lastTimestamp: "2026-07-01T18:42:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:19Z" name: splitter-graph-fbcf2-865d699f8c-46bjm.18be3e95908e624a namespace: kserve-ci-e2e-test resourceVersion: "28048" uid: fe675d83-a555-4e67-bb42-7af77757bb42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-fbcf2-865d699f8c namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: 12c8e182-9193-4b1b-bcbe-1fbdaf8c42f1 kind: Event lastTimestamp: "2026-07-01T18:41:41Z" message: 'Created pod: splitter-graph-fbcf2-865d699f8c-46bjm' metadata: creationTimestamp: "2026-07-01T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:41Z" name: splitter-graph-fbcf2-865d699f8c.18be3e929f688b03 namespace: kserve-ci-e2e-test resourceVersion: "27557" uid: 08641df7-0ad3-4409-9692-06a06d9873d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "27549" uid: e5eb1523-6faa-4c13-81c8-28790f37b161 kind: Event lastTimestamp: "2026-07-01T18:41:41Z" message: Scaled up replica set splitter-graph-fbcf2-865d699f8c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:41Z" name: splitter-graph-fbcf2.18be3e929ebe547f namespace: kserve-ci-e2e-test resourceVersion: "27554" uid: 9ed661f4-8fe7-47ce-8f14-eb401105d8b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "27569" uid: 65b10b70-f73f-4876-b3c1-8180b14e9a3a kind: Event lastTimestamp: "2026-07-01T18:41:42Z" message: InferenceGraph [splitter-graph-fbcf2] is Ready metadata: creationTimestamp: "2026-07-01T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:42Z" name: splitter-graph-fbcf2.18be3e92a4824db8 namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 2c9e24d4-59d9-4267-8b3c-dcc1b8a224e9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:30:16.910997Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df3202616e4 namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 9c46cc68-6a41-41a7-86cb-f833859a8b02 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df33aa3f4c4 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: f66ae87d-aeb7-4521-8a70-8a3ca098cf05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df33bdeeb75 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: d4182cb4-7990-4d75-8949-0abe2b876ec4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df33e7a7a05 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: ce4bb8c0-7fbf-4642-844a-32ebd4224a9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df33f5888e1 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: e9c4c23a-77f4-49f0-83fc-af31842e9804 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df33f607b8e namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 8c84d461-426b-4ce6-9fb7-d0720ad4b7f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df341fa370c namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: 5df30b30-7120-4856-ba6c-dc0e73f99575 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:17Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df3426300fa namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: f3413355-97bd-49ea-995d-e7ebb3ac7d7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:30:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:53Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3df3462018d0 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: 9a3bde08-49f8-4c76-ae4a-a0628e2f61c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3e0378177575 namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: 1fa44e86-cfa6-4d7d-ab9a-6c4ac578b628 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3e03781800b6 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 362807fc-0b9f-4396-b87f-a7644e6f8d22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: d720426e-8bce-4003-bc5a-36fa7a4a3349 kind: Event lastTimestamp: "2026-07-01T18:31:28Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:28Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6.18be3e03ce6cbd16 namespace: kserve-ci-e2e-test resourceVersion: "22259" uid: 44635e47-b7e1-4049-a9fe-efdb4ac5379d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0d3ed-predictor-65947cd9b6 namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: 5e0153a2-8e44-4a00-b852-d47abd209f25 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: 'Created pod: success-200-isvc-0d3ed-predictor-65947cd9b6-48tp6' metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-0d3ed-predictor-65947cd9b6.18be3df31f39632b namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 55be59b3-4ec5-4730-95fe-b52d2cd10bc6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: d388d1d6-0aa6-40cf-83a6-b63f6d44d10c kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: Scaled up replica set success-200-isvc-0d3ed-predictor-65947cd9b6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-0d3ed-predictor.18be3df319964be4 namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: af0528c0-2481-4749-9eb0-57faade5a412 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: ffb0d685-9f8a-4ba0-87f0-1db297ba558c kind: Event lastTimestamp: "2026-07-01T18:30:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:31Z" name: success-200-isvc-0d3ed-predictor.18be3df69811563e namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 87e287a4-68a0-4a18-9ff0-9af5a7602a9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: ffb0d685-9f8a-4ba0-87f0-1db297ba558c kind: Event lastTimestamp: "2026-07-01T18:30:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:30:31Z" name: success-200-isvc-0d3ed-predictor.18be3df698793ee6 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: c4586e8e-dca0-4c4b-921b-5a7085465f15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:30:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: ffb0d685-9f8a-4ba0-87f0-1db297ba558c kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:30:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:16Z" name: success-200-isvc-0d3ed-predictor.18be3dfa1727fab5 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: d24ced42-0f2f-46da-bcb9-9267369eb59b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:30:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0d3ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: ffb0d685-9f8a-4ba0-87f0-1db297ba558c kind: Event lastTimestamp: "2026-07-01T18:31:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:30:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:16Z" name: success-200-isvc-0d3ed-predictor.18be3dfa179c6914 namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 707256d7-f920-426a-96a2-044479420e20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 91daffcf-c450-4869-8f55-b19920c57433 kind: Event lastTimestamp: "2026-07-01T18:31:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:03Z" name: success-200-isvc-0d3ed.18be3df319967e33 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: c541e0c8-c57f-4021-a814-727724bf15a8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 91daffcf-c450-4869-8f55-b19920c57433 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-0d3ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0d3ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-0d3ed.18be3df319ea2062 namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: c2496ed9-5237-42a4-994d-c7c0e0039e6d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 91daffcf-c450-4869-8f55-b19920c57433 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0d3ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-0d3ed.18be3df319ea5a7c namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: c7b5b0a6-ba83-468b-ad3c-168425679247 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0d3ed namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 91daffcf-c450-4869-8f55-b19920c57433 kind: Event lastTimestamp: "2026-07-01T18:31:03Z" message: InferenceService [success-200-isvc-0d3ed] is Ready metadata: creationTimestamp: "2026-07-01T18:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:03Z" name: success-200-isvc-0d3ed.18be3dfe01269606 namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: cad6c0aa-0db3-48a4-84ae-b9ac02d70f1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:41:52.215275Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27723" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e9503835c43 namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 4a139c7b-7275-42d2-8ad4-67960b3cb2bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27733" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Add eth0 [10.134.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e951e6a9e92 namespace: kserve-ci-e2e-test resourceVersion: "27788" uid: f029a473-9f9d-4bc6-822a-afa8cd14abd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e951fa9a97e namespace: kserve-ci-e2e-test resourceVersion: "27790" uid: e5e4872a-b9f1-4f42-ad9f-4f6f3c7fc6bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e952292fef1 namespace: kserve-ci-e2e-test resourceVersion: "27791" uid: ebd83b2f-edd4-4f53-aa5f-6adaf26aaa4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e9522fd498f namespace: kserve-ci-e2e-test resourceVersion: "27792" uid: 54615ba6-8b91-4124-ac19-5503949e2729 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e952305f325 namespace: kserve-ci-e2e-test resourceVersion: "27793" uid: a2750490-3e16-4cf2-9712-55c8737166c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e9525a085c1 namespace: kserve-ci-e2e-test resourceVersion: "27794" uid: 0edd17a2-f36e-4fd8-9ee8-a4eb113e96bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e952609ad26 namespace: kserve-ci-e2e-test resourceVersion: "27795" uid: a4e324d5-0573-4c13-87e1-500f4f7c7fa7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:42:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:30Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3e9593880a33 namespace: kserve-ci-e2e-test resourceVersion: "28141" uid: e5a02603-9b8a-484b-b347-a8de4d15dfe8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:51:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:51:07Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3f16390f0b71 namespace: kserve-ci-e2e-test resourceVersion: "31889" uid: 3478aa57-d2dc-4e63-964c-cc19a57a23f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: baa56398-d202-4968-82d2-b81cc5563f0f kind: Event lastTimestamp: "2026-07-01T18:51:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:51:07Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq.18be3f16390f787a namespace: kserve-ci-e2e-test resourceVersion: "31891" uid: fba7d76f-384b-4dba-9abe-a766a34546e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1f8d1-predictor-6f4bc64b9d namespace: kserve-ci-e2e-test resourceVersion: "27707" uid: 8e51e4c6-feec-4dfe-8d03-08bad950d940 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'Created pod: success-200-isvc-1f8d1-predictor-6f4bc64b9d-2dvmq' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor-6f4bc64b9d.18be3e95030dce8d namespace: kserve-ci-e2e-test resourceVersion: "27725" uid: f9b8fbd3-8609-401b-bf87-621c074d8570 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27700" uid: d8740d70-24cb-4e15-82d0-84010c064eb4 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Scaled up replica set success-200-isvc-1f8d1-predictor-6f4bc64b9d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1-predictor.18be3e9501229a62 namespace: kserve-ci-e2e-test resourceVersion: "27712" uid: 1b9ff363-ddad-4326-9ad7-910ac613b648 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27706" uid: c4278721-fc04-4055-96f5-c6ebe1ba04a2 kind: Event lastTimestamp: "2026-07-01T18:42:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:37Z" name: success-200-isvc-1f8d1-predictor.18be3e988039ffed namespace: kserve-ci-e2e-test resourceVersion: "28201" uid: 1763f5c0-3c50-473a-b706-8b1e8cc8329b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27706" uid: c4278721-fc04-4055-96f5-c6ebe1ba04a2 kind: Event lastTimestamp: "2026-07-01T18:42:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:37Z" name: success-200-isvc-1f8d1-predictor.18be3e98807c92ef namespace: kserve-ci-e2e-test resourceVersion: "28202" uid: 567891d4-e352-4734-8acb-80d5c917bb45 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27940" uid: c4278721-fc04-4055-96f5-c6ebe1ba04a2 kind: Event lastTimestamp: "2026-07-01T18:43:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:07Z" name: success-200-isvc-1f8d1-predictor.18be3ea2fd333bb8 namespace: kserve-ci-e2e-test resourceVersion: "28532" uid: 02de6552-36ba-4800-88c5-f30f97da6a5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f8d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27940" uid: c4278721-fc04-4055-96f5-c6ebe1ba04a2 kind: Event lastTimestamp: "2026-07-01T18:43:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:07Z" name: success-200-isvc-1f8d1-predictor.18be3ea2fdb0fd42 namespace: kserve-ci-e2e-test resourceVersion: "28533" uid: 6d67d4dd-6071-488a-9438-65f0ce4fb0b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: a07c4f7e-4dac-45ca-b393-0f9106c5d190 kind: Event lastTimestamp: "2026-07-01T18:42:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:40Z" name: success-200-isvc-1f8d1.18be3e95010c5a4a namespace: kserve-ci-e2e-test resourceVersion: "28281" uid: 0a78330f-a6f7-4cfe-98d5-d26cbee8eaf4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: a07c4f7e-4dac-45ca-b393-0f9106c5d190 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-1f8d1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1f8d1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1.18be3e9501e0cabb namespace: kserve-ci-e2e-test resourceVersion: "27719" uid: 4c4c652f-2b61-400c-bbf3-2eb7b0ca1f12 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: a07c4f7e-4dac-45ca-b393-0f9106c5d190 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1f8d1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-1f8d1.18be3e9501e11df3 namespace: kserve-ci-e2e-test resourceVersion: "27720" uid: a738ee74-2865-4b7c-83ad-84bd9a028fa9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f8d1 namespace: kserve-ci-e2e-test resourceVersion: "28279" uid: a07c4f7e-4dac-45ca-b393-0f9106c5d190 kind: Event lastTimestamp: "2026-07-01T18:42:40Z" message: InferenceService [success-200-isvc-1f8d1] is Ready metadata: creationTimestamp: "2026-07-01T18:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:40Z" name: success-200-isvc-1f8d1.18be3ea04e73f854 namespace: kserve-ci-e2e-test resourceVersion: "28280" uid: b1b21e7d-514f-4201-8b30-205b2eabfff1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:21:02.089026Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2588a-predictor-748cbcd5cd-52g7s to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d71f22ea51e namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: e05f566c-2052-44f3-8c2f-9fff6cae0b48 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2588a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d71fef0658a namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: 594f214d-a428-49e7-a25b-557ae905c3a3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d722fcc64cb namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: ae33d9cd-2108-4cb6-ada0-8d59a52ff48d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d723120c9a2 namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: ca5d2e00-e11f-4f56-83f6-2d10c8338868 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d72348c8c1c namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: 7e749dca-0691-433e-adf9-368d07f534a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d7234f31d21 namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 4ce8cc40-fbbf-44b0-a3b5-f749f58d44f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d7234fd692d namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: a0e9c383-4cca-4e03-a2b7-00f271ba7090 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d7237f2a931 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: f7fb3360-a3c6-41b1-bcb9-34c65332dafc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:03Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d723883fab3 namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: f92ca2a1-5886-4452-a39e-4497310ef556 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:21:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:41Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3d72a51a9122 namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 82ba8947-15e2-45f8-a65a-e3d21ca4c8eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3df321b6f493 namespace: kserve-ci-e2e-test resourceVersion: "21303" uid: 30953277-32ce-4259-ae64-d767e9aad564 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: d63a87c1-08fd-4f19-8191-3f90d1eb4669 kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:16Z" name: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s.18be3df321bb72e8 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 5c6e101e-5cf0-436e-8df9-3d0071b5b657 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2588a-predictor-748cbcd5cd namespace: kserve-ci-e2e-test resourceVersion: "16872" uid: 71153ffe-8b83-4a43-bf18-beee90d493c6 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'Created pod: success-200-isvc-2588a-predictor-748cbcd5cd-52g7s' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-2588a-predictor-748cbcd5cd.18be3d71f1c49563 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 3f3eb4b0-a6a9-4d07-b1a3-0901ca78db6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: c35fa752-2f29-4a36-8811-bd9a58e85292 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: Scaled up replica set success-200-isvc-2588a-predictor-748cbcd5cd from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-2588a-predictor.18be3d71ee391552 namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: 281e3e3e-e595-4616-bfa3-8a37d3e95b60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16870" uid: 36569289-02c0-4b6c-9660-968dc48591e4 kind: Event lastTimestamp: "2026-07-01T18:21:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:17Z" name: success-200-isvc-2588a-predictor.18be3d756d0ded4a namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: fa5ce39f-f031-41de-a43e-1012d310de41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16870" uid: 36569289-02c0-4b6c-9660-968dc48591e4 kind: Event lastTimestamp: "2026-07-01T18:21:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:17Z" name: success-200-isvc-2588a-predictor.18be3d756d6d5411 namespace: kserve-ci-e2e-test resourceVersion: "17106" uid: ed2bca5b-f3d4-4dee-a025-693c06ad24f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:21:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 36569289-02c0-4b6c-9660-968dc48591e4 kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:17Z" name: success-200-isvc-2588a-predictor.18be3d78ec344bed namespace: kserve-ci-e2e-test resourceVersion: "17866" uid: ae653030-a4a2-4a41-9216-b6dae883d03c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:21:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2588a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 36569289-02c0-4b6c-9660-968dc48591e4 kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:17Z" name: success-200-isvc-2588a-predictor.18be3d78eca3ca95 namespace: kserve-ci-e2e-test resourceVersion: "17868" uid: 93d1a55b-3b74-4bfe-9f00-e45d805d859e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 3a414248-4fab-4712-aefc-2ac7e5c77785 kind: Event lastTimestamp: "2026-07-01T18:21:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:51Z" name: success-200-isvc-2588a.18be3d71ee8ebf76 namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: 94b0f8d1-aeee-4656-9506-20617a00820b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 3a414248-4fab-4712-aefc-2ac7e5c77785 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-2588a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2588a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-2588a.18be3d71ef0e8e21 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 863b1959-b513-4d4f-81e7-44a9d582a25d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 3a414248-4fab-4712-aefc-2ac7e5c77785 kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2588a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-2588a.18be3d71ef0eea48 namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: f35c68b1-6e89-4bf6-864f-22fd85ec8619 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2588a namespace: kserve-ci-e2e-test resourceVersion: "17446" uid: 3a414248-4fab-4712-aefc-2ac7e5c77785 kind: Event lastTimestamp: "2026-07-01T18:21:51Z" message: InferenceService [success-200-isvc-2588a] is Ready metadata: creationTimestamp: "2026-07-01T18:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:21:51Z" name: success-200-isvc-2588a.18be3d7d60447a70 namespace: kserve-ci-e2e-test resourceVersion: "17447" uid: a91789c4-6510-4ca2-be87-8e6db3a61138 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:32:22.976611Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22829" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-293b3-predictor-67c5856c84-z6lcg to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:32:22Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e107a3ea745 namespace: kserve-ci-e2e-test resourceVersion: "22835" uid: 78a06987-fc6f-46c5-b5f0-1fbeb6114a47 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22846" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e1093f33787 namespace: kserve-ci-e2e-test resourceVersion: "22885" uid: 933260de-da19-429a-a5a6-47e198136608 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e109547be3f namespace: kserve-ci-e2e-test resourceVersion: "22887" uid: ef2e4da2-dfec-49ad-95a4-df3cfcb4804f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e109879f4b0 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 3c761ffb-fc74-49b0-bfa1-723bac64bab2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e1098e14ead namespace: kserve-ci-e2e-test resourceVersion: "22890" uid: 6de1cfc0-9326-4966-8fc3-5e541afcbe88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e1098e9817a namespace: kserve-ci-e2e-test resourceVersion: "22891" uid: b4d77c42-c785-4fe5-8a0f-7f352128414c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e109bf9b694 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: 6f0011bd-5b39-43cf-9bd6-bfe3b8d127b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:32:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:23Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e109c63c555 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: eb6c551a-fb93-44f2-bf58-f0508f3a68db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:41:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:40Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e10bf8414c9 namespace: kserve-ci-e2e-test resourceVersion: "27520" uid: 08f4fb83-2afd-45ea-806c-b7b033b092f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:37Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e91ad60a734 namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: 1fc6516f-ca87-4b9f-a056-345983ee1267 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:37Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e91ad61d557 namespace: kserve-ci-e2e-test resourceVersion: "27433" uid: 24946df6-e61c-4297-8247-56c29e185cac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 7962bc6c-a206-4599-a2da-a43cc1669814 kind: Event lastTimestamp: "2026-07-01T18:41:40Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:40Z" name: success-200-isvc-293b3-predictor-67c5856c84-z6lcg.18be3e9233fa2be0 namespace: kserve-ci-e2e-test resourceVersion: "27519" uid: 92be93ad-4660-4d5c-b34f-82ccbc8c85fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-293b3-predictor-67c5856c84 namespace: kserve-ci-e2e-test resourceVersion: "22810" uid: d9faaebd-f1d1-4c95-a1f4-7fa7b2f61bc4 kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: 'Created pod: success-200-isvc-293b3-predictor-67c5856c84-z6lcg' metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:22Z" name: success-200-isvc-293b3-predictor-67c5856c84.18be3e1079c7b8ae namespace: kserve-ci-e2e-test resourceVersion: "22830" uid: e98dfce2-93f6-4dc3-b41e-a28c03e0ded5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "22807" uid: caaf45d9-9052-4542-b869-c7112c8173ec kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: Scaled up replica set success-200-isvc-293b3-predictor-67c5856c84 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:22Z" name: success-200-isvc-293b3-predictor.18be3e1077ceb83a namespace: kserve-ci-e2e-test resourceVersion: "22813" uid: c138afb5-28dd-48b1-84f6-727a40134f85 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "22808" uid: f5632b01-d6a0-492a-974b-0eb3d1612720 kind: Event lastTimestamp: "2026-07-01T18:32:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:37Z" name: success-200-isvc-293b3-predictor.18be3e13f74c97b1 namespace: kserve-ci-e2e-test resourceVersion: "23072" uid: f8b341d3-52ac-4c74-800d-93c0d8b90593 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "22808" uid: f5632b01-d6a0-492a-974b-0eb3d1612720 kind: Event lastTimestamp: "2026-07-01T18:32:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:37Z" name: success-200-isvc-293b3-predictor.18be3e13f79872c7 namespace: kserve-ci-e2e-test resourceVersion: "23073" uid: 906d4efa-7611-4d6e-a421-a47e3c7528ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:32:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23071" uid: f5632b01-d6a0-492a-974b-0eb3d1612720 kind: Event lastTimestamp: "2026-07-01T18:33:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:53Z" name: success-200-isvc-293b3-predictor.18be3e17765e02f0 namespace: kserve-ci-e2e-test resourceVersion: "23687" uid: d781063e-8580-426e-bf26-b7a2b9cc157e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:32:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-293b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23071" uid: f5632b01-d6a0-492a-974b-0eb3d1612720 kind: Event lastTimestamp: "2026-07-01T18:33:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:33:53Z" name: success-200-isvc-293b3-predictor.18be3e1776d6e086 namespace: kserve-ci-e2e-test resourceVersion: "23689" uid: b4ca0cb2-babf-4e6b-8364-50da9b982bd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-293b3 namespace: kserve-ci-e2e-test resourceVersion: "22789" uid: c8123690-0759-40df-964a-348fee53bf33 kind: Event lastTimestamp: "2026-07-01T18:33:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:10Z" name: success-200-isvc-293b3.18be3e1077dba715 namespace: kserve-ci-e2e-test resourceVersion: "23340" uid: 96531059-2613-48b2-9067-ae3f6616b07f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:33:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-293b3 namespace: kserve-ci-e2e-test resourceVersion: "23337" uid: c8123690-0759-40df-964a-348fee53bf33 kind: Event lastTimestamp: "2026-07-01T18:33:10Z" message: InferenceService [success-200-isvc-293b3] is Ready metadata: creationTimestamp: "2026-07-01T18:33:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:33:10Z" name: success-200-isvc-293b3.18be3e1b7b3c623f namespace: kserve-ci-e2e-test resourceVersion: "23338" uid: 3e6df57a-9293-492f-9839-61574a714f04 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:19:27.573799Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:19:27Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5bf0a2b2f9 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 89c3213d-5139-43f6-a877-886bd6124a87 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:19:28Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5c0cca4654 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: fd2a2150-df78-40ef-be52-279012295512 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:28Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" metadata: creationTimestamp: "2026-07-01T18:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:28Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5c0e5276f1 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: c2aa7e4c-f476-4eca-acd0-c7924c142607 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:40Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" in 12.899s (12.899s including waiting). Image size: 1335676004 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:40Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5f0f3333da namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: f0d8e202-e841-4804-a110-5ad0daf58652 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5f12abc9e9 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: e082c8cd-2921-44be-90db-800d4d55415a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5f131e04ba namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 7a2f13ab-f3e7-4b2d-91ce-3f99fed69824 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:41Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5f132ac16a namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: eb34b6fc-f0d2-4462-beb6-bca065b72e71 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.253s (3.253s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:44Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5fd51b4051 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 7ff7fe9e-8e77-4298-9c6e-6cbd2a79bc6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:44Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5fd8a1ce88 namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: 2e1cbcd9-f87b-4b39-900f-231562f3b71a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:19:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:44Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5fd917611f namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: f42a36f7-642d-4cec-9e0b-fe9928748fc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:20:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:20Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d5fed286eca namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 18e2faf4-c84c-4d3d-ab59-84a12a004fd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d71ef2bb4f1 namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: c9e064db-b915-461e-b8dc-637b4ea8713e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: ce57eb2b-fcf3-4a6a-8f78-3a164b55582e kind: Event lastTimestamp: "2026-07-01T18:21:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:02Z" name: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd.18be3d71ef315803 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 207b6989-a601-4aa7-b6b5-8727fbad99b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4d812-predictor-6f5465f7b8 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 690a6308-1785-485b-90a4-6c31a100f8e2 kind: Event lastTimestamp: "2026-07-01T18:19:27Z" message: 'Created pod: success-200-isvc-4d812-predictor-6f5465f7b8-s2kqd' metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:27Z" name: success-200-isvc-4d812-predictor-6f5465f7b8.18be3d5bf016534f namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: bca0194c-47a2-46d2-94aa-1882c4a3aa00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 896582b0-7614-47e2-899f-21f24ef39569 kind: Event lastTimestamp: "2026-07-01T18:19:27Z" message: Scaled up replica set success-200-isvc-4d812-predictor-6f5465f7b8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:27Z" name: success-200-isvc-4d812-predictor.18be3d5becbe0544 namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 39bea833-2018-4df3-bd7e-ba2af89fddaf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: ebbc67b3-ad03-4c2a-9e71-84562812152b kind: Event lastTimestamp: "2026-07-01T18:20:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:12Z" name: success-200-isvc-4d812-predictor.18be3d5f722783a8 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: 0650483b-7e2c-4521-a75b-aa241f1d5f1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: ebbc67b3-ad03-4c2a-9e71-84562812152b kind: Event lastTimestamp: "2026-07-01T18:20:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:12Z" name: success-200-isvc-4d812-predictor.18be3d5f72aafdc4 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 490550d8-b2c3-4250-af3d-56d7c874a5df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:19:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: ebbc67b3-ad03-4c2a-9e71-84562812152b kind: Event lastTimestamp: "2026-07-01T18:20:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:42Z" name: success-200-isvc-4d812-predictor.18be3d62f16af7bd namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 9b27eed8-ed4a-40b0-b198-d2b937aab0a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:19:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d812-predictor namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: ebbc67b3-ad03-4c2a-9e71-84562812152b kind: Event lastTimestamp: "2026-07-01T18:20:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:42Z" name: success-200-isvc-4d812-predictor.18be3d62f1d58afc namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 12298c94-14dc-4c15-8bfe-f7519a4afe8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:19:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4d812 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: d3886016-532c-498f-9081-4aed0d3dd8d1 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:30Z" name: success-200-isvc-4d812.18be3d5bf2cd1c85 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: d90a0d07-477c-4f89-85fb-ac6b1912f431 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4d812 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: d3886016-532c-498f-9081-4aed0d3dd8d1 kind: Event lastTimestamp: "2026-07-01T18:20:30Z" message: InferenceService [success-200-isvc-4d812] is Ready metadata: creationTimestamp: "2026-07-01T18:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:30Z" name: success-200-isvc-4d812.18be3d6aa8f9e8cb namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 15115e27-681b-492c-8d2f-ecbbee2db972 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:41:37.885433Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27422" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-60085-predictor-5b6dd5d96d-gltrl to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:41:37Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91ad634d9c namespace: kserve-ci-e2e-test resourceVersion: "27427" uid: 6778b26a-6d27-4fd0-8ecd-cbec9588ebe0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91c76d60c4 namespace: kserve-ci-e2e-test resourceVersion: "27481" uid: 900e2ff5-b6ea-480d-a2cb-726907dc40ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91c8f4d1ff namespace: kserve-ci-e2e-test resourceVersion: "27483" uid: 902701a6-6fba-4fd5-8629-408109b7d3d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91cbe059c3 namespace: kserve-ci-e2e-test resourceVersion: "27484" uid: 540759a2-43bd-4cd5-9fc5-5e133c582a44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91cc653029 namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: fbc5522f-7cf7-41f7-b065-356ef62aaccf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91cc6d32e0 namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: 845a141c-863f-4383-8ce5-12e77780f679 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91cf38113d namespace: kserve-ci-e2e-test resourceVersion: "27488" uid: 9001f537-309d-4aec-81c4-e71a2d2d425c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:41:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:38Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91cf9ef7e4 namespace: kserve-ci-e2e-test resourceVersion: "27489" uid: 182eb8c9-5c61-4d04-836d-e8dc3b0d5b21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:42:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:14Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3e91d4dc714f namespace: kserve-ci-e2e-test resourceVersion: "28012" uid: 997b8e90-ddf1-48b0-8f04-d676401d44d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:58:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:57Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3f83afd19121 namespace: kserve-ci-e2e-test resourceVersion: "35159" uid: 6ab87d70-bfe6-4247-b84c-2b413606249b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 56af3c3a-ceb8-4dad-9d0a-9cc3071b056e kind: Event lastTimestamp: "2026-07-01T18:58:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:57Z" name: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl.18be3f83afd1f228 namespace: kserve-ci-e2e-test resourceVersion: "35160" uid: d6bfbdb4-53c3-48fe-bc87-0ac15d823231 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-60085-predictor-5b6dd5d96d namespace: kserve-ci-e2e-test resourceVersion: "27406" uid: 5d14fad8-ac47-4077-85dd-6f6421c0ac39 kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: 'Created pod: success-200-isvc-60085-predictor-5b6dd5d96d-gltrl' metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:37Z" name: success-200-isvc-60085-predictor-5b6dd5d96d.18be3e91acee229f namespace: kserve-ci-e2e-test resourceVersion: "27424" uid: b3508aca-57e1-498d-ba96-e160981ce3aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: 089eb20d-8b42-44ad-89e6-3390c33bcf6f kind: Event lastTimestamp: "2026-07-01T18:41:37Z" message: Scaled up replica set success-200-isvc-60085-predictor-5b6dd5d96d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:37Z" name: success-200-isvc-60085-predictor.18be3e91ab118654 namespace: kserve-ci-e2e-test resourceVersion: "27411" uid: d08b6d6e-d1c7-472d-a4a5-7b7cfd84e0b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27404" uid: 1ba92177-4662-49b7-80c8-9d537e93fda8 kind: Event lastTimestamp: "2026-07-01T18:42:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:22Z" name: success-200-isvc-60085-predictor.18be3e952a2c6a8b namespace: kserve-ci-e2e-test resourceVersion: "28082" uid: 5051c95e-37d0-407a-b007-2e6d4fecb576 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27404" uid: 1ba92177-4662-49b7-80c8-9d537e93fda8 kind: Event lastTimestamp: "2026-07-01T18:42:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:22Z" name: success-200-isvc-60085-predictor.18be3e952aa1c0e8 namespace: kserve-ci-e2e-test resourceVersion: "28083" uid: 0ce95e3b-4ebc-4210-bd52-28c25a3d30c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: 1ba92177-4662-49b7-80c8-9d537e93fda8 kind: Event lastTimestamp: "2026-07-01T18:42:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:52Z" name: success-200-isvc-60085-predictor.18be3e98a9dff849 namespace: kserve-ci-e2e-test resourceVersion: "28413" uid: d8c867b0-9807-4671-ba5e-97f342ff863b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60085-predictor namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: 1ba92177-4662-49b7-80c8-9d537e93fda8 kind: Event lastTimestamp: "2026-07-01T18:42:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:52Z" name: success-200-isvc-60085-predictor.18be3e98aa6fd468 namespace: kserve-ci-e2e-test resourceVersion: "28414" uid: 93ca1dcc-2aa3-4f5f-a2b0-71145c8eed03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:41:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60085 namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: 8e49e778-fbbe-4582-ab8b-fda5ebfa0735 kind: Event lastTimestamp: "2026-07-01T18:42:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:24Z" name: success-200-isvc-60085.18be3e91ab423d33 namespace: kserve-ci-e2e-test resourceVersion: "28106" uid: 8d609e57-62b4-4a67-a655-ceac34fc469e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60085 namespace: kserve-ci-e2e-test resourceVersion: "28103" uid: 8e49e778-fbbe-4582-ab8b-fda5ebfa0735 kind: Event lastTimestamp: "2026-07-01T18:42:24Z" message: InferenceService [success-200-isvc-60085] is Ready metadata: creationTimestamp: "2026-07-01T18:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:24Z" name: success-200-isvc-60085.18be3e9c902b4246 namespace: kserve-ci-e2e-test resourceVersion: "28104" uid: 297a1bc8-94fd-46c8-bbfa-1756a50e5e39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:31:12.728119Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9b590-predictor-7674b6fd4d-hdktq to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e001f1bac0a namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: d385b270-7cc3-428e-8638-7ba161816735 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9b590-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e002b0c3748 namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 9aad6bba-3433-4302-a4dd-0e059aaf8d24 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e005c3e63a9 namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: d2644435-4057-44d1-adf4-58e682e2dd07 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e005d8fe239 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 285e0ae4-31b5-4ea0-9f20-f67948f9ca9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e006095af0b namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 9ea8f01b-9fb6-444b-a399-99dcb369c89a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e00610118b2 namespace: kserve-ci-e2e-test resourceVersion: "21940" uid: f4191efe-fece-41f4-b6f0-cda60ab21069 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e00610a100b namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: f322a9c4-6415-401e-ae65-7badb92c98de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e0063d29e10 namespace: kserve-ci-e2e-test resourceVersion: "21943" uid: b0f4171e-35de-4f3e-8761-084e1c921a60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:13Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e00643a4695 namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: a2c01a1f-dd6f-46b0-8f45-0b49eee28abf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:31:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:50Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e009b1189ee namespace: kserve-ci-e2e-test resourceVersion: "22455" uid: 4ced565b-e5a7-4af2-aa5d-1beade38d5e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:22Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e107aaa2121 namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 391a5f12-81ed-453b-ba04-b6de996ec886 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 756ceb0c-6b74-4ceb-b1a6-4461f1dd4cf6 kind: Event lastTimestamp: "2026-07-01T18:32:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:32:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:32:22Z" name: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq.18be3e107aaa988e namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: 5b288e38-a34c-4802-bfd6-7abb61efe4a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9b590-predictor-7674b6fd4d namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 3ae2a87a-2fed-4563-86ff-45b5b5c19c4e kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'Created pod: success-200-isvc-9b590-predictor-7674b6fd4d-hdktq' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-9b590-predictor-7674b6fd4d.18be3e001eb236cc namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 828c616c-bbed-4557-8478-900a8239ee95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: aded5129-58aa-43fb-a658-895466184246 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Scaled up replica set success-200-isvc-9b590-predictor-7674b6fd4d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-9b590-predictor.18be3e001ca106d0 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: bf98e77d-9410-4f9d-a030-da7a568b3a17 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: ca3dbfc9-4dae-4a75-b1f8-19df9da111a4 kind: Event lastTimestamp: "2026-07-01T18:31:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:42Z" name: success-200-isvc-9b590-predictor.18be3e039b31e0bf namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 1f9331ad-1c11-463d-99c4-4aba70822f84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: ca3dbfc9-4dae-4a75-b1f8-19df9da111a4 kind: Event lastTimestamp: "2026-07-01T18:31:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:42Z" name: success-200-isvc-9b590-predictor.18be3e039b9c8a07 namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: 45fb1472-0c8a-4fa3-a39c-110bbe005595 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: ca3dbfc9-4dae-4a75-b1f8-19df9da111a4 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:12Z" name: success-200-isvc-9b590-predictor.18be3e0a992ed309 namespace: kserve-ci-e2e-test resourceVersion: "22655" uid: 9505819d-c6aa-429c-9d31-d60df50ba4f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9b590-predictor namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: ca3dbfc9-4dae-4a75-b1f8-19df9da111a4 kind: Event lastTimestamp: "2026-07-01T18:32:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:12Z" name: success-200-isvc-9b590-predictor.18be3e0a999d42fe namespace: kserve-ci-e2e-test resourceVersion: "22659" uid: 295508b4-6cca-418f-997d-8eb08f44088f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 26ce3f78-7765-4b53-9861-c19c31e35984 kind: Event lastTimestamp: "2026-07-01T18:32:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:00Z" name: success-200-isvc-9b590.18be3e001cc9fcb8 namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 74552c52-83d8-49aa-9f41-d72e4eae580a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 26ce3f78-7765-4b53-9861-c19c31e35984 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-9b590": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9b590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-9b590.18be3e001d378d76 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: af84b0aa-28cb-408d-9a0a-7e6fb199880b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 26ce3f78-7765-4b53-9861-c19c31e35984 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9b590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-9b590.18be3e001d37bee7 namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: 933d54bf-f9d9-4184-b121-1c74e05ff418 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9b590 namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: 26ce3f78-7765-4b53-9861-c19c31e35984 kind: Event lastTimestamp: "2026-07-01T18:32:00Z" message: InferenceService [success-200-isvc-9b590] is Ready metadata: creationTimestamp: "2026-07-01T18:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:00Z" name: success-200-isvc-9b590.18be3e0b56907f3f namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: 37859bca-1564-449a-9b3d-07d9bf8f566e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:31:27.118733Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e0378db054c namespace: kserve-ci-e2e-test resourceVersion: "22183" uid: 85783c37-b866-4faa-aa52-bcdc638b60f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22185" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e039253a701 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 5451dc80-3c24-43a1-b05d-5286e157c086 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e0393b7017c namespace: kserve-ci-e2e-test resourceVersion: "22225" uid: a85a63fb-ebba-4488-9aab-3051596deb16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e0396e23837 namespace: kserve-ci-e2e-test resourceVersion: "22226" uid: 02edea90-d479-4730-813f-f511d41be8ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e03975348f3 namespace: kserve-ci-e2e-test resourceVersion: "22227" uid: a02b4101-ae71-4630-9a05-e6d0ecc9d4b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e03975effcb namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: f01d78aa-60ac-49e5-9fdf-19585eb45892 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e039a0c101f namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 12cbb39e-1d1c-4bcf-b87d-d494af1647f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e039a78bf47 namespace: kserve-ci-e2e-test resourceVersion: "22232" uid: 21234099-f895-4258-b5a0-7c43a3a70da9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e03a5e734fa namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: 2aaed14a-495e-4575-982c-66688db67f86 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e84a5cfb22a namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: 494b4570-1ab3-4d9a-8f1f-2d341aa1c9b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e84a5d0262a namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 2585cee2-903f-41c9-9601-431e2f63cb43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: a64ad961-1fb4-468f-9acc-4e72cc202d9a kind: Event lastTimestamp: "2026-07-01T18:40:43Z" message: 'Readiness probe failed: Get "https://10.134.0.46:8643/healthz": dial tcp 10.134.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:43Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq.18be3e851a682ebc namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: 27eab652-2bdf-4d46-832c-81451b6e7157 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c1eb4-predictor-6547fdb46d namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 9fcaab36-bc3e-4275-9de1-94896761d89d kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: 'Created pod: success-200-isvc-c1eb4-predictor-6547fdb46d-zh9tq' metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor-6547fdb46d.18be3e0378335f49 namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: 4c856cef-b369-419e-96c9-234c50fa1c86 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22149" uid: 64c25fa8-8d4f-46cd-98f3-ed99bb21eeac kind: Event lastTimestamp: "2026-07-01T18:31:27Z" message: Scaled up replica set success-200-isvc-c1eb4-predictor-6547fdb46d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:31:27Z" name: success-200-isvc-c1eb4-predictor.18be3e037659a5a8 namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 236949ab-4137-4b97-b335-8f94b933bbe2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: d6398aa4-b14f-4178-9b19-8e3644f5cc09 kind: Event lastTimestamp: "2026-07-01T18:32:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:42Z" name: success-200-isvc-c1eb4-predictor.18be3e06f58263c4 namespace: kserve-ci-e2e-test resourceVersion: "23115" uid: 190ee38d-a645-4763-93ad-2c6dc6147564 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:31:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1eb4-predictor namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: d6398aa4-b14f-4178-9b19-8e3644f5cc09 kind: Event lastTimestamp: "2026-07-01T18:32:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:32:42Z" name: success-200-isvc-c1eb4-predictor.18be3e06f5f12f8d namespace: kserve-ci-e2e-test resourceVersion: "23117" uid: 6305a245-64c9-4e45-a44c-beb84c977c94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: ede25e44-58f7-4d79-8dc3-f65a316e8c09 kind: Event lastTimestamp: "2026-07-01T18:32:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:14Z" name: success-200-isvc-c1eb4.18be3e03765128c9 namespace: kserve-ci-e2e-test resourceVersion: "22696" uid: bc51d352-405b-424c-b7c1-0f73c0521d63 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:32:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1eb4 namespace: kserve-ci-e2e-test resourceVersion: "22693" uid: ede25e44-58f7-4d79-8dc3-f65a316e8c09 kind: Event lastTimestamp: "2026-07-01T18:32:13Z" message: InferenceService [success-200-isvc-c1eb4] is Ready metadata: creationTimestamp: "2026-07-01T18:32:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:32:13Z" name: success-200-isvc-c1eb4.18be3e0e61731583 namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: a8c90a5b-52d8-4b02-bde0-7066f540a8f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:21:57.910684Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17566" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:21:57Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7ef16a865f namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: 543e77aa-cb81-4e61-9ea5-fd6fc68bdb17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c5c10-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:58Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7efc49e952 namespace: kserve-ci-e2e-test resourceVersion: "17624" uid: 04c69906-264b-48c5-bd37-3e7e16967e2e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17590" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:58Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:21:58Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f2e871796 namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: 480c7abc-76dc-4f23-9ee7-a9070c629ada reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:58Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:58Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f2fc140b4 namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: acdccc5a-44ea-4fd1-b236-ccb3e89ea4c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f324fe6ce namespace: kserve-ci-e2e-test resourceVersion: "17644" uid: b4d89a69-f23a-43c3-a7f3-6b79db2e0e3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f32ab5451 namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: ba3a7550-ea6b-4dab-ad6e-2b3054217515 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f32b40bad namespace: kserve-ci-e2e-test resourceVersion: "17646" uid: c5e1aca1-9411-4443-964b-63ac61d7f913 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f35144000 namespace: kserve-ci-e2e-test resourceVersion: "17647" uid: 6af75009-5649-4651-b723-e60a0a8b276f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:21:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:59Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f356dd16d namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: 39a67195-6b29-4534-9e05-5bb55d797267 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:22:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:35Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3d7f47b2225d namespace: kserve-ci-e2e-test resourceVersion: "18008" uid: aa738013-3439-4b4b-bc76-2d5771111cd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3e001d9c0e2e namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: 9801104b-5c35-4814-baab-e6dad21c3efa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 2f3f6391-8a2a-4d4c-8d28-ed8d9843d2b2 kind: Event lastTimestamp: "2026-07-01T18:31:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:31:12Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf.18be3e001d9cc259 namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: d37c7f71-5fa3-42bf-9faa-ff511fe082f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c5c10-predictor-dfd5bc4db namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: a029c722-5680-4b2b-99de-b01ef0cd5c31 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: 'Created pod: success-200-isvc-c5c10-predictor-dfd5bc4db-rf4mf' metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:57Z" name: success-200-isvc-c5c10-predictor-dfd5bc4db.18be3d7ef0f1080a namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: e41f20b4-1105-4adc-9280-e4457b930d70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: c6ac3e7d-e3f6-4b76-8de6-77aed7d9cb08 kind: Event lastTimestamp: "2026-07-01T18:21:57Z" message: Scaled up replica set success-200-isvc-c5c10-predictor-dfd5bc4db from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:57Z" name: success-200-isvc-c5c10-predictor.18be3d7eef4ae6f2 namespace: kserve-ci-e2e-test resourceVersion: "17558" uid: 8e44bae4-e3c1-4541-a55c-d548cd662e5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17552" uid: 353e97fc-5b6e-44af-902e-ff4218036766 kind: Event lastTimestamp: "2026-07-01T18:22:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:27Z" name: success-200-isvc-c5c10-predictor.18be3d826e670746 namespace: kserve-ci-e2e-test resourceVersion: "17951" uid: 1f5a49c4-8167-4aa6-9375-1774741e911f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17552" uid: 353e97fc-5b6e-44af-902e-ff4218036766 kind: Event lastTimestamp: "2026-07-01T18:22:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:27Z" name: success-200-isvc-c5c10-predictor.18be3d826f07cc5f namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: 32f64e3f-afe0-44d8-8813-10c9af90d6aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17827" uid: 353e97fc-5b6e-44af-902e-ff4218036766 kind: Event lastTimestamp: "2026-07-01T18:22:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:57Z" name: success-200-isvc-c5c10-predictor.18be3d896d4e0b90 namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: 71d5ad78-4eb8-421e-8f74-6d00cb0f06d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:22:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c5c10-predictor namespace: kserve-ci-e2e-test resourceVersion: "17827" uid: 353e97fc-5b6e-44af-902e-ff4218036766 kind: Event lastTimestamp: "2026-07-01T18:22:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:57Z" name: success-200-isvc-c5c10-predictor.18be3d896db78cdb namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: d9b591e2-2037-41c6-b2bd-aeeb61ac6088 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "17523" uid: 8b915266-a393-44ae-aeda-9dcc450ced02 kind: Event lastTimestamp: "2026-07-01T18:22:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:45Z" name: success-200-isvc-c5c10.18be3d7eef483f1f namespace: kserve-ci-e2e-test resourceVersion: "18097" uid: b49a0a54-7082-4e7a-8c7c-baafda256863 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c5c10 namespace: kserve-ci-e2e-test resourceVersion: "18094" uid: 8b915266-a393-44ae-aeda-9dcc450ced02 kind: Event lastTimestamp: "2026-07-01T18:22:45Z" message: InferenceService [success-200-isvc-c5c10] is Ready metadata: creationTimestamp: "2026-07-01T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:45Z" name: success-200-isvc-c5c10.18be3d8a02e9c149 namespace: kserve-ci-e2e-test resourceVersion: "18095" uid: b75ad54a-79ab-49b8-a9f7-dc7a8fe25380 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:40:41.929351Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fbcf2-predictor-56df98589b-4qbbb to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84a6254a04 namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 816cb1b7-a2ca-4dc5-8e61-d48fe71c0efd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c02ec454 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: 7bcb0917-95a1-4a62-859b-4d5ae076d06f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c181ea86 namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: b49102dd-1eb6-4de0-b330-007a3cae378b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c48bc24a namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: ed8e67d4-2c55-45bc-9181-4b579d79a5d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c50259d2 namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: f1fe9552-be8e-418d-83ad-59870717c4ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c50bf98f namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: d79e9ace-eba7-49ad-ad56-dc101c35400a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c8019cf9 namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: 953f9a93-7956-43c7-bef7-bf84f02d93cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:40:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:40:42Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84c868e5bc namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: 9a491aba-df02-4b03-999a-86e524e73be8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:41:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:19Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e84f7db0f45 namespace: kserve-ci-e2e-test resourceVersion: "27212" uid: 97ac51c9-47ab-4240-8e74-2106c3854ed4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e950260fc4d namespace: kserve-ci-e2e-test resourceVersion: "27722" uid: 6c42a8b0-6fef-41d6-8bd8-aa9457001605 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:41:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:52Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e950261a714 namespace: kserve-ci-e2e-test resourceVersion: "27724" uid: 35622ccf-e3ee-4c8d-93bb-affaa5ffae5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 196a5e57-de4b-4d01-b2ef-f2a573144ea5 kind: Event lastTimestamp: "2026-07-01T18:41:54Z" message: 'Readiness probe failed: Get "https://10.134.0.51:8643/healthz": dial tcp 10.134.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:41:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:41:54Z" name: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb.18be3e957ffad49a namespace: kserve-ci-e2e-test resourceVersion: "27823" uid: a09b7cf3-70e9-4eff-85e3-04ef97406569 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fbcf2-predictor-56df98589b namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: a0e39397-0162-4618-a95e-46bb556b6ffa kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: 'Created pod: success-200-isvc-fbcf2-predictor-56df98589b-4qbbb' metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-fbcf2-predictor-56df98589b.18be3e84a5da64a5 namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: 3e68f4be-0587-4820-bff2-4fa246d7e6ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: a8984ef6-6909-4b40-862f-72b22daa9f7a kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: Scaled up replica set success-200-isvc-fbcf2-predictor-56df98589b from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-fbcf2-predictor.18be3e84a21ef22e namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: d2c843f7-61d2-4ae2-9cb3-361b70fdbc12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:40:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: a4122328-0105-4af9-aec5-585aa7dbc33b kind: Event lastTimestamp: "2026-07-01T18:41:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:11Z" name: success-200-isvc-fbcf2-predictor.18be3e88207bde32 namespace: kserve-ci-e2e-test resourceVersion: "27155" uid: a58ebc48-a53e-41f1-82d0-1604e1d89d79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:40:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: a4122328-0105-4af9-aec5-585aa7dbc33b kind: Event lastTimestamp: "2026-07-01T18:41:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:11Z" name: success-200-isvc-fbcf2-predictor.18be3e8821129283 namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: e59a2cc0-9e6e-492e-9353-464445701825 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:41:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: a4122328-0105-4af9-aec5-585aa7dbc33b kind: Event lastTimestamp: "2026-07-01T18:41:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:41Z" name: success-200-isvc-fbcf2-predictor.18be3e8f1ec039f0 namespace: kserve-ci-e2e-test resourceVersion: "27550" uid: 9d9ff682-cc67-40ed-9ba9-bab4c0e70be6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:41:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbcf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: a4122328-0105-4af9-aec5-585aa7dbc33b kind: Event lastTimestamp: "2026-07-01T18:41:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:41:41Z" name: success-200-isvc-fbcf2-predictor.18be3e8f1f4a1af6 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: 99946279-f757-4b92-a0f1-a9e731de8f82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 1785bd77-4c66-4ac4-9992-cbf43f003697 kind: Event lastTimestamp: "2026-07-01T18:41:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:29Z" name: success-200-isvc-fbcf2.18be3e84a21d11a6 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: 5088cf9f-b6aa-4c62-a4fe-f5747a07a9cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 1785bd77-4c66-4ac4-9992-cbf43f003697 kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-fbcf2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbcf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-fbcf2.18be3e84a2788164 namespace: kserve-ci-e2e-test resourceVersion: "26816" uid: 14c03612-20fe-4556-a265-6fee766673fb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 1785bd77-4c66-4ac4-9992-cbf43f003697 kind: Event lastTimestamp: "2026-07-01T18:40:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbcf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:40:41Z" name: success-200-isvc-fbcf2.18be3e84a278cb3c namespace: kserve-ci-e2e-test resourceVersion: "26817" uid: 568594a3-16d3-40d3-8045-2a7c9e43b23c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbcf2 namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: 1785bd77-4c66-4ac4-9992-cbf43f003697 kind: Event lastTimestamp: "2026-07-01T18:41:29Z" message: InferenceService [success-200-isvc-fbcf2] is Ready metadata: creationTimestamp: "2026-07-01T18:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:41:29Z" name: success-200-isvc-fbcf2.18be3e8fb76c70aa namespace: kserve-ci-e2e-test resourceVersion: "27297" uid: eeec997a-7401-4c0b-bf2d-2ff10086cf00 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:22:02.111772Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-2588a-79864c4578-t2p5b to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:22:02Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3d7febd0faf2 namespace: kserve-ci-e2e-test resourceVersion: "17713" uid: 5575c14a-f7fe-42d5-a4e7-f609826d22fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:22:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-2588a-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:02Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3d7ff8471d0d namespace: kserve-ci-e2e-test resourceVersion: "17728" uid: 91b40a29-2882-4685-9c6c-980caf9c3c80 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:03Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17717" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:22:03Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:22:03Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3d8028c4fb25 namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: 1a774660-5dbb-4ed6-a4bd-563d31130387 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2588a} kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:22:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:03Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3d8029f6ec4e namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: 536ec53e-6126-4af3-9bac-3a2648b5ebf4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2588a} kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:22:03Z" message: 'Created container: switch-graph-2588a' metadata: creationTimestamp: "2026-07-01T18:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:03Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3d802d92fa40 namespace: kserve-ci-e2e-test resourceVersion: "17746" uid: 396c6dbe-8daf-474c-a6b0-5bdad7702d46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2588a} kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:22:03Z" message: Started container switch-graph-2588a metadata: creationTimestamp: "2026-07-01T18:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:22:03Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3d802df64c1d namespace: kserve-ci-e2e-test resourceVersion: "17747" uid: ec00a664-73bf-4df4-9757-2e7cb949c164 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2588a} kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:30:16Z" message: Stopping container switch-graph-2588a metadata: creationTimestamp: "2026-07-01T18:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:16Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3df31311169e namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 76dd527b-cf1a-4966-ae1c-645230c044ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:30:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2588a} kind: Pod name: switch-graph-2588a-79864c4578-t2p5b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: ee45e391-e73c-4dd3-8759-494ede34c0dc kind: Event lastTimestamp: "2026-07-01T18:30:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:30:44Z" name: switch-graph-2588a-79864c4578-t2p5b.18be3df3b3adaf63 namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 798140f4-32c7-4204-afc1-fd46fad762a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-2588a-79864c4578 namespace: kserve-ci-e2e-test resourceVersion: "17703" uid: 839c6194-40ba-41e2-b3b7-8e1913dac793 kind: Event lastTimestamp: "2026-07-01T18:22:02Z" message: 'Created pod: switch-graph-2588a-79864c4578-t2p5b' metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:02Z" name: switch-graph-2588a-79864c4578.18be3d7feb635817 namespace: kserve-ci-e2e-test resourceVersion: "17709" uid: f0313d86-f138-47eb-b063-4ceeaf4ae45c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-2588a namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 40f626b1-c662-43a1-8231-ae6f84bbabbe kind: Event lastTimestamp: "2026-07-01T18:22:02Z" message: Scaled up replica set switch-graph-2588a-79864c4578 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:02Z" name: switch-graph-2588a.18be3d7feaaab839 namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: b15dca05-903a-4a26-bfd3-be6a7d26b1ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-2588a namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: 41134ba1-4ded-4c1c-b2de-18dfc902d0fb kind: Event lastTimestamp: "2026-07-01T18:22:02Z" message: InferenceGraph [switch-graph-2588a] is Ready metadata: creationTimestamp: "2026-07-01T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:22:02Z" name: switch-graph-2588a.18be3d7ff062f857 namespace: kserve-ci-e2e-test resourceVersion: "17724" uid: 6364701f-4242-4354-89fe-72179c8e48a2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2588a namespace: kserve-ci-e2e-test resourceVersion: "17702" uid: f07edcdd-a602-4aa4-b5c9-2df093dacf4a kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:17Z" name: switch-graph-2588a.18be3d8369440816 namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 29db2cba-8d76-4c1b-b49b-85182a391cd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2588a namespace: kserve-ci-e2e-test resourceVersion: "17702" uid: f07edcdd-a602-4aa4-b5c9-2df093dacf4a kind: Event lastTimestamp: "2026-07-01T18:22:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:17Z" name: switch-graph-2588a.18be3d8369ae784a namespace: kserve-ci-e2e-test resourceVersion: "17870" uid: eedad14a-f9bf-4ba8-b890-efd719f59618 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2588a namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: f07edcdd-a602-4aa4-b5c9-2df093dacf4a kind: Event lastTimestamp: "2026-07-01T18:22:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:32Z" name: switch-graph-2588a.18be3d86e86d2022 namespace: kserve-ci-e2e-test resourceVersion: "17986" uid: 6fec5423-ed62-4661-bbbb-c4c8f21ad9c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:22:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-2588a namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: f07edcdd-a602-4aa4-b5c9-2df093dacf4a kind: Event lastTimestamp: "2026-07-01T18:22:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:22:32Z" name: switch-graph-2588a.18be3d86e8bcedca namespace: kserve-ci-e2e-test resourceVersion: "17989" uid: 0982e166-cf2f-4bf5-9728-66e9be53c6df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:20:47.646201Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4d812-656c86c8f8-ggh6z to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:20:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:20:47Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6e95529fbe namespace: kserve-ci-e2e-test resourceVersion: "16690" uid: be2e42a1-d3ee-42b0-a9f0-ef62bb4f2ed2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:20:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4d812-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:47Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6e9d2a045f namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 4162bf00-10a6-4f37-9402-8dfbc0960ef7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:20:48Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:20:48Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6ed2bb9521 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: fd401893-7e28-44b9-b495-02967346c470 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4d812} kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:20:48Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" metadata: creationTimestamp: "2026-07-01T18:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:48Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6ed3e5e7f9 namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: 72e59097-615e-45ee-833b-e1fb9a03b385 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4d812} kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:20:52Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" in 3.385s (3.385s including waiting). Image size: 218070118 bytes.' metadata: creationTimestamp: "2026-07-01T18:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:52Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6f9da9ed15 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: f3ae5b0e-3666-44d0-b707-de80ac5892aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4d812} kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:20:52Z" message: 'Created container: switch-graph-4d812' metadata: creationTimestamp: "2026-07-01T18:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:52Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6fa0d19186 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 461ddabf-dfc1-48cd-b497-9e894aa80717 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4d812} kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:20:52Z" message: Started container switch-graph-4d812 metadata: creationTimestamp: "2026-07-01T18:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:52Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d6fa138eb01 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 1d03c7a5-ed00-4e3a-a836-958afd7374c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:21:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4d812} kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:21:01Z" message: Stopping container switch-graph-4d812 metadata: creationTimestamp: "2026-07-01T18:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:01Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d71e62df8c6 namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: 7300b2a2-393e-42b3-abae-421c55e587bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4d812} kind: Pod name: switch-graph-4d812-656c86c8f8-ggh6z namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7c4b5f0c-abed-4bc9-abf9-36062c41567a kind: Event lastTimestamp: "2026-07-01T18:21:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:21:29Z" name: switch-graph-4d812-656c86c8f8-ggh6z.18be3d7265fa6979 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 59be6696-7307-4f63-ac8d-4dd0848978df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4d812-656c86c8f8 namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: c32b93fe-daaa-4bdc-9c06-dfb5b9d40297 kind: Event lastTimestamp: "2026-07-01T18:20:47Z" message: 'Created pod: switch-graph-4d812-656c86c8f8-ggh6z' metadata: creationTimestamp: "2026-07-01T18:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:47Z" name: switch-graph-4d812-656c86c8f8.18be3d6e94e8c3cd namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: 27617a42-7272-4c2b-8868-19f9667b5c70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4d812 namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 09766350-b0be-479a-ae58-a28356d04b4f kind: Event lastTimestamp: "2026-07-01T18:20:47Z" message: Scaled up replica set switch-graph-4d812-656c86c8f8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:47Z" name: switch-graph-4d812.18be3d6e93f611c0 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: faab5f4a-e194-41a0-8b3c-364229a4b5fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4d812 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 11479a99-a77a-4978-91ee-02ea8c31c18c kind: Event lastTimestamp: "2026-07-01T18:20:47Z" message: InferenceGraph [switch-graph-4d812] is Ready metadata: creationTimestamp: "2026-07-01T18:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:47Z" name: switch-graph-4d812.18be3d6e97e6b9e2 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: e3d474cd-87a9-4504-9cc5-b45d5f6faf80 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:42:37.949865Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28220" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-60085-86b755c89-djvlx to ip-10-0-136-171.ec2.internal metadata: creationTimestamp: "2026-07-01T18:42:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:42:37Z" name: switch-graph-60085-86b755c89-djvlx.18be3e9fa981cadc namespace: kserve-ci-e2e-test resourceVersion: "28227" uid: 0fc2d343-7aba-4d89-be8a-4787d7213d69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4674fc8c-xzjfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28222" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:42:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-60085-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:38Z" name: switch-graph-60085-86b755c89-djvlx.18be3e9fb0a65561 namespace: kserve-ci-e2e-test resourceVersion: "28242" uid: ba218d57-90b8-4ba1-83bf-8cadf04792e2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28230" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:42:38Z" message: Add eth0 [10.134.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:42:38Z" name: switch-graph-60085-86b755c89-djvlx.18be3e9fe64ec941 namespace: kserve-ci-e2e-test resourceVersion: "28251" uid: 4f3d15a3-2417-44c8-89b4-88d81667517e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60085} kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28222" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:42:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:38Z" name: switch-graph-60085-86b755c89-djvlx.18be3e9fe79abbf3 namespace: kserve-ci-e2e-test resourceVersion: "28254" uid: e91e7d6f-9d4d-4fbc-a035-3f9eaeb83368 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60085} kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28222" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:42:39Z" message: 'Created container: switch-graph-60085' metadata: creationTimestamp: "2026-07-01T18:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:39Z" name: switch-graph-60085-86b755c89-djvlx.18be3e9fea8393ba namespace: kserve-ci-e2e-test resourceVersion: "28256" uid: c18d2f59-45cb-48b1-9a2f-920ed3e8ea10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60085} kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28222" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:42:39Z" message: Started container switch-graph-60085 metadata: creationTimestamp: "2026-07-01T18:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:42:39Z" name: switch-graph-60085-86b755c89-djvlx.18be3e9feaf27f67 namespace: kserve-ci-e2e-test resourceVersion: "28257" uid: f3a351ee-50f9-4fb8-a027-9b4b0bb5393f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60085} kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28222" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:58:57Z" message: Stopping container switch-graph-60085 metadata: creationTimestamp: "2026-07-01T18:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:57Z" name: switch-graph-60085-86b755c89-djvlx.18be3f83a5f1299c namespace: kserve-ci-e2e-test resourceVersion: "35143" uid: 0e5766dd-7c79-4669-9d14-6f6adbcde1f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:59:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60085} kind: Pod name: switch-graph-60085-86b755c89-djvlx namespace: kserve-ci-e2e-test resourceVersion: "28222" uid: 3066e176-0337-4751-a566-a7995aac8c14 kind: Event lastTimestamp: "2026-07-01T18:59:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:20Z" name: switch-graph-60085-86b755c89-djvlx.18be3f8482ceec5f namespace: kserve-ci-e2e-test resourceVersion: "35387" uid: fc970265-feb1-4e1b-b197-8ccd6e7e1663 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-171.ec2.internal source: component: kubelet host: ip-10-0-136-171.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-60085-86b755c89 namespace: kserve-ci-e2e-test resourceVersion: "28217" uid: fa34be8e-5a9e-492e-9039-03e2dd2d77d5 kind: Event lastTimestamp: "2026-07-01T18:42:37Z" message: 'Created pod: switch-graph-60085-86b755c89-djvlx' metadata: creationTimestamp: "2026-07-01T18:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:37Z" name: switch-graph-60085-86b755c89.18be3e9fa9404f90 namespace: kserve-ci-e2e-test resourceVersion: "28224" uid: 812119a4-3dd2-45d9-8d74-90e818d0e7db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-60085 namespace: kserve-ci-e2e-test resourceVersion: "28215" uid: eb29aa3b-5593-482f-bcef-4cfdd26fe786 kind: Event lastTimestamp: "2026-07-01T18:42:37Z" message: Scaled up replica set switch-graph-60085-86b755c89 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:42:37Z" name: switch-graph-60085.18be3e9fa825595c namespace: kserve-ci-e2e-test resourceVersion: "28219" uid: 34ffa8fd-0ec2-45a9-b454-0720508faff5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:42:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-60085 namespace: kserve-ci-e2e-test resourceVersion: "28237" uid: 31389f67-45be-44ce-9f62-8f2be8649d7d kind: Event lastTimestamp: "2026-07-01T18:42:38Z" message: InferenceGraph [switch-graph-60085] is Ready metadata: creationTimestamp: "2026-07-01T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:42:38Z" name: switch-graph-60085.18be3e9faf01c608 namespace: kserve-ci-e2e-test resourceVersion: "28238" uid: 8826ba8f-46bc-451d-b625-174645706dfa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-60085 namespace: kserve-ci-e2e-test resourceVersion: "28216" uid: 3ef3912c-10ab-492a-9e7f-ecef8464c4dd kind: Event lastTimestamp: "2026-07-01T18:43:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:22Z" name: switch-graph-60085.18be3ea326986274 namespace: kserve-ci-e2e-test resourceVersion: "28647" uid: c11178c2-5d9a-4d37-8345-d04d10a6addb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:42:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-60085 namespace: kserve-ci-e2e-test resourceVersion: "28216" uid: 3ef3912c-10ab-492a-9e7f-ecef8464c4dd kind: Event lastTimestamp: "2026-07-01T18:43:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:43:22Z" name: switch-graph-60085.18be3ea327078b2c namespace: kserve-ci-e2e-test resourceVersion: "28648" uid: 619a77ea-be3d-4b0a-9f74-0762092bd6ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "35408"