--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:00.780635Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19072" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-226e9-596665bfbf-p64qh to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:00Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef6b7ac666d namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: 448d25e2-0547-45fb-90d6-56f582a15917 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-226e9-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:00Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef6c20dfedd namespace: kserve-ci-e2e-test resourceVersion: "19091" uid: 056ea175-115f-4b33-81e1-ab653614c6cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19081" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:01Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef6f455e761 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: be7129ca-a29e-4461-8aea-66669b786d79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-226e9} kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:01Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef6f5b2c9eb namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: e4fcdf0a-b813-4550-8709-6aee465086b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-226e9} kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: 'Created container: ensemble-graph-226e9' metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:01Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef6f8e9b66c namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: 077ec92f-a898-4b14-9e26-c52cfc3b3eab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-226e9} kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: Started container ensemble-graph-226e9 metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:01Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef6f9591566 namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: bb261241-64f0-47cd-ad04-fb8cb5831ec5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-226e9} kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Stopping container ensemble-graph-226e9 metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:10Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef90f5f80f1 namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: feaa9501-f73d-46be-ba9d-e12ff8ba4a11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-226e9} kind: Pod name: ensemble-graph-226e9-596665bfbf-p64qh namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5fe1e4c4-4461-4367-9e98-75a1e61bc178 kind: Event lastTimestamp: "2026-04-23T18:13:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:38Z" name: ensemble-graph-226e9-596665bfbf-p64qh.18a90ef9bb5b22c0 namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: 27d2bb0a-fbbc-4df7-a8cd-a2438d404777 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-226e9-596665bfbf namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: c7c4779a-f867-498f-a0da-e74ff22c3630 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: 'Created pod: ensemble-graph-226e9-596665bfbf-p64qh' metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:00Z" name: ensemble-graph-226e9-596665bfbf.18a90ef6b73d9bf2 namespace: kserve-ci-e2e-test resourceVersion: "19073" uid: 204559c0-37d8-4aed-811e-bcd7a3c472f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-226e9 namespace: kserve-ci-e2e-test resourceVersion: "19067" uid: bb00e08b-b4cb-4572-8507-0f859a9ff842 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: Scaled up replica set ensemble-graph-226e9-596665bfbf from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:00Z" name: ensemble-graph-226e9.18a90ef6b6a4e928 namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 5a1696ce-3637-42a1-adc2-cfe1310e43c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-226e9 namespace: kserve-ci-e2e-test resourceVersion: "19083" uid: 357aa4d2-f425-4dea-bd14-343cf64f24b8 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: InferenceGraph [ensemble-graph-226e9] is Ready metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:00Z" name: ensemble-graph-226e9.18a90ef6ba52b03b namespace: kserve-ci-e2e-test resourceVersion: "19085" uid: 1a2ec424-0e05-477d-ad97-8c386f682bd7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:14:11.066800Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5c6fb-d586947bb-nqqrr to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:14:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:14:11Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f07150e29ed namespace: kserve-ci-e2e-test resourceVersion: "20036" uid: cb8c6ab6-b389-4dce-9b32-f13354bbb55a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:11Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:14:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-5c6fb-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:11Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f071bf6322c namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 726a84cb-3f03-41e0-9eb3-a41d4d8e962b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:12Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:14:12Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:14:12Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f0751836c6c namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: fd0b4687-2db6-4f48-b1d7-6d353682b890 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c6fb} kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:14:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:12Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f0753078eb0 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 54a97ce1-a8c1-4421-807c-17b38cef1de3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c6fb} kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:14:12Z" message: 'Created container: ensemble-graph-5c6fb' metadata: creationTimestamp: "2026-04-23T18:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:12Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f0755a0b3e3 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 38611d0e-d22f-443c-88a5-9ffe18695606 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c6fb} kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:14:12Z" message: Started container ensemble-graph-5c6fb metadata: creationTimestamp: "2026-04-23T18:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:12Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f0756117af7 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 49444cae-a9de-467f-8b1d-5c2640b7d2ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c6fb} kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:22:25Z" message: Stopping container ensemble-graph-5c6fb metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:25Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f7a471bfea6 namespace: kserve-ci-e2e-test resourceVersion: "23548" uid: 295edca6-419a-492c-98fe-cedabfd12fb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c6fb} kind: Pod name: ensemble-graph-5c6fb-d586947bb-nqqrr namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6afaa16e-b778-4ab1-89c7-155606b721da kind: Event lastTimestamp: "2026-04-23T18:22:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:53Z" name: ensemble-graph-5c6fb-d586947bb-nqqrr.18a90f7aff81d1db namespace: kserve-ci-e2e-test resourceVersion: "23889" uid: 21d3e0af-7430-478f-8008-ee90e5d1dabf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5c6fb-d586947bb namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 04b1d03b-4d58-448c-ab6c-8ed2e1460d0f kind: Event lastTimestamp: "2026-04-23T18:14:11Z" message: 'Created pod: ensemble-graph-5c6fb-d586947bb-nqqrr' metadata: creationTimestamp: "2026-04-23T18:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:11Z" name: ensemble-graph-5c6fb-d586947bb.18a90f071497943d namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 762a5b22-9fa5-4cfe-8452-2ccc5ad09728 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 4c3105f0-a7c3-4964-9eae-3c0c1611b7cd kind: Event lastTimestamp: "2026-04-23T18:14:11Z" message: Scaled up replica set ensemble-graph-5c6fb-d586947bb from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:11Z" name: ensemble-graph-5c6fb.18a90f0713ef26b0 namespace: kserve-ci-e2e-test resourceVersion: "20029" uid: bdb56ca2-ec65-43d8-8a5b-0e4efbaaec86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: c39a51cb-634c-4500-a6e7-a55611f2f0aa kind: Event lastTimestamp: "2026-04-23T18:14:11Z" message: InferenceGraph [ensemble-graph-5c6fb] is Ready metadata: creationTimestamp: "2026-04-23T18:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:11Z" name: ensemble-graph-5c6fb.18a90f071ad3e451 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: d39b05a2-25ff-42fa-8d2d-4475e7c44f3b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:14:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: e053f61d-b0c4-480b-9970-c55326345524 kind: Event lastTimestamp: "2026-04-23T18:14:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:56Z" name: ensemble-graph-5c6fb.18a90f0a926f2d7d namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: 7f19be7d-8305-4441-93f6-1128c9f8255d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:14:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: e053f61d-b0c4-480b-9970-c55326345524 kind: Event lastTimestamp: "2026-04-23T18:14:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:56Z" name: ensemble-graph-5c6fb.18a90f0a92b7f004 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: a7c9683b-c416-4089-8965-f36ff1d4169a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:03:25.971931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0065f-predictor-585bf4fc7-ff7tf to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:03:25Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e70e26795fb namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: e619e7ed-b883-4cd5-819b-ec4fee51ffdc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0065f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:26Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e70ec6d6edc namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 51d59398-ebfe-4386-8ef5-a5ac55d6d919 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e711fb8ba09 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: d1905067-239e-41aa-b71e-b08138f64bad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e7121052cec namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 48fe3724-b262-49ce-8e00-f5c1923a2536 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e71241e5b05 namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: dbc608b1-90c3-4be5-b3c1-4ec10af879f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e7124a04f20 namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: f5de4aa7-d3ae-4d16-b671-2539599577e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e7124ad2c93 namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 1a1f29d2-c688-46ea-922c-e89dee24ac4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e712876f6c7 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 8f696e9e-7002-4da2-8ea4-a7e65ffa5bd5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e7128eddd4a namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: c41144ca-6b92-4610-adab-5d16c0c84205 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:04:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:04Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90e7184c03434 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 3816ecb0-b1d9-4659-846d-9f3c7616ab4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90ef20f5b12e3 namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: f1fb21d3-80f1-4771-8adf-0e2c68160918 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 1e3a4970-c2f9-4767-9862-9e8cc797ccc3 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf.18a90ef20f5c5aa4 namespace: kserve-ci-e2e-test resourceVersion: "18817" uid: e5eb1996-bb04-4b40-a6b5-f55ea4fa44f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0065f-predictor-585bf4fc7 namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 24e09412-149e-423b-ae67-0f363ab29035 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: 'Created pod: error-404-isvc-0065f-predictor-585bf4fc7-ff7tf' metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:25Z" name: error-404-isvc-0065f-predictor-585bf4fc7.18a90e70e1ec8cfe namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 99a89050-73ac-4eb8-b993-a54482366b96 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 94a89377-0f3f-43f6-8abf-f9d180d945ea kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Scaled up replica set error-404-isvc-0065f-predictor-585bf4fc7 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:25Z" name: error-404-isvc-0065f-predictor.18a90e70dfebbf96 namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 56a8156e-c9f2-4db5-b2bf-132931a3b75a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 06be90e7-979c-45d6-9ad3-144d6c9ed8a7 kind: Event lastTimestamp: "2026-04-23T18:03:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:55Z" name: error-404-isvc-0065f-predictor.18a90e745e6860ad namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 1b50b8c2-396d-4f9f-b40a-2b10805c0eda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 06be90e7-979c-45d6-9ad3-144d6c9ed8a7 kind: Event lastTimestamp: "2026-04-23T18:03:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:55Z" name: error-404-isvc-0065f-predictor.18a90e745ec5a46a namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 05e7cb19-6f58-490d-a5bd-265d96e5b986 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 06be90e7-979c-45d6-9ad3-144d6c9ed8a7 kind: Event lastTimestamp: "2026-04-23T18:04:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:41Z" name: error-404-isvc-0065f-predictor.18a90e7b5cdd0fe5 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 505bd71e-719b-4194-9290-b4cede9a271c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 06be90e7-979c-45d6-9ad3-144d6c9ed8a7 kind: Event lastTimestamp: "2026-04-23T18:04:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:41Z" name: error-404-isvc-0065f-predictor.18a90e7b5d3cde01 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: e6a70717-85e0-42ce-a97e-5d9e70be5404 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0065f namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 354c0d2d-6952-4b02-9439-925a096de90e kind: Event lastTimestamp: "2026-04-23T18:04:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:14Z" name: error-404-isvc-0065f.18a90e70dfa54bb1 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 8cbe03d3-fac6-4088-9dd6-4f00496fe07f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0065f namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 354c0d2d-6952-4b02-9439-925a096de90e kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-0065f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0065f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:25Z" name: error-404-isvc-0065f.18a90e70e01f5e5f namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e901eff5-de1f-4fba-b94a-885725d26401 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0065f namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 354c0d2d-6952-4b02-9439-925a096de90e kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0065f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:25Z" name: error-404-isvc-0065f.18a90e70e01faa95 namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: b4ff7989-9aa7-4db6-8f8a-29b7b938553a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0065f namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 354c0d2d-6952-4b02-9439-925a096de90e kind: Event lastTimestamp: "2026-04-23T18:04:14Z" message: InferenceService [error-404-isvc-0065f] is Ready metadata: creationTimestamp: "2026-04-23T18:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:14Z" name: error-404-isvc-0065f.18a90e7c40f97e59 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 67653761-3b35-4b87-a50b-f34d8f45e9e7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:00.811169Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-226e9-predictor-76db5f4c75-z7mjh to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee8c136f6f6 namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: c81ea72f-7ee7-444f-afd4-8d136fb2cef1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-226e9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee8c82aad7d namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: 43c15e07-3614-42d4-952e-13640dd1c4b3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee8fe4c8280 namespace: kserve-ci-e2e-test resourceVersion: "18416" uid: d7adabee-07a2-4051-b91a-e715db0c0124 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee9004946bb namespace: kserve-ci-e2e-test resourceVersion: "18418" uid: abcfe59d-c55f-4e38-a448-bd5c9aa9d680 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee903795c79 namespace: kserve-ci-e2e-test resourceVersion: "18420" uid: 34347df1-1536-410b-a9ee-2cce59f9432e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee904055e1c namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bc291f1f-75ca-4472-b032-364609ba8d45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee904110ff3 namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: c09f167c-a387-48a6-abe0-39c33ddc4dd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee906d6a12f namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 51e7526c-3e09-44b8-a63c-69b16e26b369 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee90764a314 namespace: kserve-ci-e2e-test resourceVersion: "18425" uid: a8dd769c-b76c-47aa-9b02-9e6a8bd7224d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:12:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:39Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ee962f7990d namespace: kserve-ci-e2e-test resourceVersion: "18726" uid: 3edf0ef4-3bb9-41ff-8a9c-d63c79f241b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ef91b178d60 namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: 07a0985e-1649-4589-b642-d8865a7aec80 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: e3581511-69fd-45a6-872b-b4273dc7d9be kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh.18a90ef91b180fca namespace: kserve-ci-e2e-test resourceVersion: "19261" uid: ec2c808e-4bfc-47f6-a93c-8f171d0c9f1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-226e9-predictor-76db5f4c75 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 7effb453-0326-48ef-b1c1-15dfe8d0167a kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'Created pod: error-404-isvc-226e9-predictor-76db5f4c75-z7mjh' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-226e9-predictor-76db5f4c75.18a90ee8c0d7ed46 namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: 05c56906-cf95-43d3-86f9-5b3ba85e44b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: e1efb66e-6128-43bb-b9bf-cfb101be91ef kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Scaled up replica set error-404-isvc-226e9-predictor-76db5f4c75 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-226e9-predictor.18a90ee8bf6c3216 namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: 52138afb-9554-4577-93f8-90e227b17a2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: 1ac243e9-7a79-4c08-ab43-cc69513915b5 kind: Event lastTimestamp: "2026-04-23T18:12:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:15Z" name: error-404-isvc-226e9-predictor.18a90eec3dfce6fe namespace: kserve-ci-e2e-test resourceVersion: "18546" uid: 8ea29991-dbcb-4786-a224-fcd2d74bee48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: 1ac243e9-7a79-4c08-ab43-cc69513915b5 kind: Event lastTimestamp: "2026-04-23T18:12:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:15Z" name: error-404-isvc-226e9-predictor.18a90eec3e6487d7 namespace: kserve-ci-e2e-test resourceVersion: "18548" uid: a957bcb9-a0a1-48e7-9404-c5c12641770d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18547" uid: 1ac243e9-7a79-4c08-ab43-cc69513915b5 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:00Z" name: error-404-isvc-226e9-predictor.18a90eefbd307a83 namespace: kserve-ci-e2e-test resourceVersion: "19086" uid: 41c01f6e-a033-4e3a-a8a1-7dcf34eb7015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18547" uid: 1ac243e9-7a79-4c08-ab43-cc69513915b5 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:00Z" name: error-404-isvc-226e9-predictor.18a90eefbd9e0fec namespace: kserve-ci-e2e-test resourceVersion: "19087" uid: 378e19ec-3db2-47c3-a978-b5cba1822de3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 5e145a15-d1d2-4eb7-8543-b42624e0f498 kind: Event lastTimestamp: "2026-04-23T18:12:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:49Z" name: error-404-isvc-226e9.18a90ee8bf4491d0 namespace: kserve-ci-e2e-test resourceVersion: "18964" uid: b1c3a089-4e6c-4550-a138-fddf01d3dd35 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 5e145a15-d1d2-4eb7-8543-b42624e0f498 kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-226e9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-226e9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-226e9.18a90ee8bf9bf220 namespace: kserve-ci-e2e-test resourceVersion: "18375" uid: c77f2305-7425-4077-86b0-a091eb26dfe2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 5e145a15-d1d2-4eb7-8543-b42624e0f498 kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-226e9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-226e9.18a90ee8bf9c407c namespace: kserve-ci-e2e-test resourceVersion: "18376" uid: 55639387-6378-4dc4-a4f2-60855cb8bece reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18962" uid: 5e145a15-d1d2-4eb7-8543-b42624e0f498 kind: Event lastTimestamp: "2026-04-23T18:12:49Z" message: InferenceService [error-404-isvc-226e9] is Ready metadata: creationTimestamp: "2026-04-23T18:12:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:49Z" name: error-404-isvc-226e9.18a90ef41e43e36a namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: 4e5dd629-ef82-42b8-98a2-284d16c4246c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:02:46.030798Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13945" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3d086-predictor-69454595bf-z64wk to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e6795ba4c0a namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: bebae43e-1bcc-4cc0-9bf5-105203c5e336 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13955" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67af599bc3 namespace: kserve-ci-e2e-test resourceVersion: "13971" uid: 5bf95001-702b-43dc-95d8-2a29b6601935 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67b0aef314 namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: 16414859-9cfa-4047-85df-36996e3e2f6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67b3fadf9a namespace: kserve-ci-e2e-test resourceVersion: "13976" uid: 0c826ee8-3486-4231-a314-e96251dad298 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67b46e6e54 namespace: kserve-ci-e2e-test resourceVersion: "13977" uid: 69fd3d3c-3e13-4ed1-8dc1-e26f2a898588 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67b47a2c10 namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: e3bdc764-1628-4034-b523-829711ce6c54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67b7b9f67a namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: 14ee5ce9-f99a-42e0-888f-74ca98c2785b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67b831dcf8 namespace: kserve-ci-e2e-test resourceVersion: "13980" uid: e12c260d-5f63-4905-8324-0ce6f09829e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:12:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:03Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90e67efb49646 namespace: kserve-ci-e2e-test resourceVersion: "18447" uid: 0a08e382-67a9-4d34-a842-a104cdc94377 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90ee8bc1f86b0 namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: 16e61bb1-93f7-477e-906f-6828d0474ee6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:00Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90ee8bc201836 namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: 545fa1bd-ee77-412e-9a3a-8a1552ccca51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d086-predictor-69454595bf-z64wk namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 6005bc75-ebb7-4cca-bc84-3cc0f0866a27 kind: Event lastTimestamp: "2026-04-23T18:12:03Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:03Z" name: error-404-isvc-3d086-predictor-69454595bf-z64wk.18a90ee9640dbcff namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: d38b5333-e52d-4c2f-9a47-0a1ca7ea19cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3d086-predictor-69454595bf namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 677c69ae-5e0b-4b9b-973f-307c99aea3bb kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'Created pod: error-404-isvc-3d086-predictor-69454595bf-z64wk' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086-predictor-69454595bf.18a90e6795341d97 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: 0a439d15-6947-4029-ac5c-019e08a457c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: b6f4e14a-d467-45e7-9269-ec086d822ae9 kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Scaled up replica set error-404-isvc-3d086-predictor-69454595bf from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:45Z" name: error-404-isvc-3d086-predictor.18a90e6793a79cb6 namespace: kserve-ci-e2e-test resourceVersion: "13940" uid: 20afd24a-b0b9-4c08-9fd4-83accf3d54ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 721aac4e-7781-4d04-a744-a54fed69b250 kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:16Z" name: error-404-isvc-3d086-predictor.18a90e6b1229e595 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: d6b59479-d819-472d-88f6-27993cee6a7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 721aac4e-7781-4d04-a744-a54fed69b250 kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:16Z" name: error-404-isvc-3d086-predictor.18a90e6b12997380 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 263336a5-6ba2-45b6-9881-db3514a13c61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:03:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: 721aac4e-7781-4d04-a744-a54fed69b250 kind: Event lastTimestamp: "2026-04-23T18:04:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:01Z" name: error-404-isvc-3d086-predictor.18a90e72102c9c66 namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: b7620893-9cd5-4819-b588-a80a7d130cce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:03:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: 721aac4e-7781-4d04-a744-a54fed69b250 kind: Event lastTimestamp: "2026-04-23T18:04:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:01Z" name: error-404-isvc-3d086-predictor.18a90e7210a4340b namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: f1a45378-8aa2-4f59-9161-53117150e5e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: b1b09a24-1efa-41fb-953e-e136c08d5607 kind: Event lastTimestamp: "2026-04-23T18:03:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:33Z" name: error-404-isvc-3d086.18a90e679380dd56 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 87c5bf62-cb30-49c9-a85c-49179401a071 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: b1b09a24-1efa-41fb-953e-e136c08d5607 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-3d086": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3d086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086.18a90e6793f15dcb namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 38455db7-1ef5-4759-a132-a8e59d9ea1e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: b1b09a24-1efa-41fb-953e-e136c08d5607 kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3d086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:46Z" name: error-404-isvc-3d086.18a90e6793f1abe0 namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: a1121e54-b500-4639-befa-289311e9f6a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: b1b09a24-1efa-41fb-953e-e136c08d5607 kind: Event lastTimestamp: "2026-04-23T18:03:33Z" message: InferenceService [error-404-isvc-3d086] is Ready metadata: creationTimestamp: "2026-04-23T18:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:33Z" name: error-404-isvc-3d086.18a90e72ab1babc7 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: dededbf4-097a-4df6-93a5-7eeea15f4758 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:11.116332Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19275" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5c6fb-predictor-7b8548d59-n54tw to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef91fbaa091 namespace: kserve-ci-e2e-test resourceVersion: "19279" uid: 1f98c6bf-523d-47be-a1e8-d19445b3f34b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19283" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef939c25c1a namespace: kserve-ci-e2e-test resourceVersion: "19301" uid: b105ce6f-95aa-4924-b6eb-bc48f4d1fb29 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef93b2d1641 namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: 36229135-101b-476a-9086-d0e9fe69b940 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef93eab1c00 namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: c14a9f33-1fe0-4ab3-bac9-a733ecf49dfc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef93f2892c4 namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: b42ec23c-3968-4a9b-8523-f22c2d2ed3a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef93f350e30 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 22eda9cb-73c8-4601-9ca9-ccdf9ae92423 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef94271381b namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: 65a96d3a-9a7e-4b20-8bf7-844b41719c0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef942edd375 namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 25d8615e-d4ad-4ba8-8a5d-db9766b9d3f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:22:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:28Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90ef983e191ec namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: 88270b14-37ff-4ee6-84cb-6d8ec7c6a60c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90f7a5387ed5c namespace: kserve-ci-e2e-test resourceVersion: "23604" uid: 17501484-3072-4986-a316-41b0eb5fa4dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90f7a53886c6f namespace: kserve-ci-e2e-test resourceVersion: "23609" uid: d64904ca-b610-40fb-9d95-e104f9bb898d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 5d12a6f2-06cb-4a1e-92bd-d77fd200a7d6 kind: Event lastTimestamp: "2026-04-23T18:22:28Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:28Z" name: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw.18a90f7af8410eec namespace: kserve-ci-e2e-test resourceVersion: "23696" uid: 4b2bd505-4818-4ff1-98d8-f7b3059fe758 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5c6fb-predictor-7b8548d59 namespace: kserve-ci-e2e-test resourceVersion: "19267" uid: 334a22b5-0d36-42af-bad9-f077d97dd3d9 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created pod: error-404-isvc-5c6fb-predictor-7b8548d59-n54tw' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor-7b8548d59.18a90ef91edf1574 namespace: kserve-ci-e2e-test resourceVersion: "19277" uid: bff5f367-855a-4056-bc6f-407e4ad1a93c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: 48382729-8bb6-4412-aa91-3b39cf703417 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Scaled up replica set error-404-isvc-5c6fb-predictor-7b8548d59 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb-predictor.18a90ef91d153a57 namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: 4655700e-55f6-4831-8d84-de2eccc0dad4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: c7f1d024-b8f4-4cd2-bbf9-fd6a9e325e70 kind: Event lastTimestamp: "2026-04-23T18:14:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:41Z" name: error-404-isvc-5c6fb-predictor.18a90efc9bbd818d namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: eb17c17b-7586-4b54-9152-e5a35254b786 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: c7f1d024-b8f4-4cd2-bbf9-fd6a9e325e70 kind: Event lastTimestamp: "2026-04-23T18:14:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:41Z" name: error-404-isvc-5c6fb-predictor.18a90efc9c30bdab namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: a4a07196-5955-4cec-83b0-3ac006fc6021 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: a07278bb-e8e2-46a6-9622-f7f0b1cc0d2b kind: Event lastTimestamp: "2026-04-23T18:13:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:58Z" name: error-404-isvc-5c6fb.18a90ef91cf0b9f5 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: b0fe0fcd-822c-4bb8-8d47-d6f061e8da48 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: a07278bb-e8e2-46a6-9622-f7f0b1cc0d2b kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-5c6fb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5c6fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb.18a90ef91d4bf8f1 namespace: kserve-ci-e2e-test resourceVersion: "19272" uid: e4dd298a-e4da-469c-bd10-b89aeab722aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: a07278bb-e8e2-46a6-9622-f7f0b1cc0d2b kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5c6fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:11Z" name: error-404-isvc-5c6fb.18a90ef91d4c5aa4 namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: 685c0951-c83e-41b8-b4de-4e85cf7d179d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: a07278bb-e8e2-46a6-9622-f7f0b1cc0d2b kind: Event lastTimestamp: "2026-04-23T18:13:58Z" message: InferenceService [error-404-isvc-5c6fb] is Ready metadata: creationTimestamp: "2026-04-23T18:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:58Z" name: error-404-isvc-5c6fb.18a90f043f49a654 namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: 25f0b580-e6df-4cd8-9535-225025d6831e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:36.405128Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24512" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7efef-predictor-86c486dbf-v2nw4 to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8ab5d8975f namespace: kserve-ci-e2e-test resourceVersion: "24516" uid: d0876623-aad8-40dd-9905-c591f49aeaaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7efef-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8ac1c536bf namespace: kserve-ci-e2e-test resourceVersion: "24526" uid: 0e3aa673-a81e-4c03-9c59-c7581c2b2304 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24520" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8af32b227e namespace: kserve-ci-e2e-test resourceVersion: "24544" uid: 352c2a1d-3590-491a-bcf3-f9082e4bda9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8af4819c90 namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 67ebbc83-320b-425b-8fa6-70760133d272 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8af866180a namespace: kserve-ci-e2e-test resourceVersion: "24551" uid: 60bf1d7c-f42f-4741-a962-0ab085becb43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8af8ddd0da namespace: kserve-ci-e2e-test resourceVersion: "24552" uid: 67ae8433-8c12-4f3a-9b90-c2a6bca84f66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8af8e9bff7 namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: cb9fab7f-fdea-44ee-a91a-d0487afbcafb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8afcba0cea namespace: kserve-ci-e2e-test resourceVersion: "24555" uid: 520d1949-f3c9-4374-8003-4da9f383a255 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8afd37bc18 namespace: kserve-ci-e2e-test resourceVersion: "24556" uid: 5afd3c15-2307-4b87-889a-5cd942f94ae7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:24:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:15Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a90f8b51bbceea namespace: kserve-ci-e2e-test resourceVersion: "24896" uid: 6e38144a-0102-4cbc-84c6-073cb683da37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:32:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:51Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a9100be69e3dd3 namespace: kserve-ci-e2e-test resourceVersion: "28919" uid: 293a36cd-394b-4ce2-a268-4739005f0957 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4 namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 1bc0f098-a86d-4d55-9c5e-1a36edbda04f kind: Event lastTimestamp: "2026-04-23T18:32:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:51Z" name: error-404-isvc-7efef-predictor-86c486dbf-v2nw4.18a9100be69f8676 namespace: kserve-ci-e2e-test resourceVersion: "28920" uid: 53c97a02-0e3b-49f5-8d9f-a248dddae19c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7efef-predictor-86c486dbf namespace: kserve-ci-e2e-test resourceVersion: "24505" uid: b01411d7-d572-437e-b62b-3d931f940d1f kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'Created pod: error-404-isvc-7efef-predictor-86c486dbf-v2nw4' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-7efef-predictor-86c486dbf.18a90f8ab54ccf1a namespace: kserve-ci-e2e-test resourceVersion: "24513" uid: 028689db-40ed-48de-bcc9-77442029c77a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24503" uid: a9f3ee86-93b6-444a-9934-50d1b8e71e0d kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Scaled up replica set error-404-isvc-7efef-predictor-86c486dbf from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-7efef-predictor.18a90f8ab3d0e8a7 namespace: kserve-ci-e2e-test resourceVersion: "24507" uid: e49506bb-e516-4b64-bd64-4cb494c9eb04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24504" uid: 92b46f90-ff88-4838-9865-841fba76e96f kind: Event lastTimestamp: "2026-04-23T18:24:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:21Z" name: error-404-isvc-7efef-predictor.18a90f8e32533884 namespace: kserve-ci-e2e-test resourceVersion: "24991" uid: d57a1723-daad-4f58-8667-d5bc7b379959 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24504" uid: 92b46f90-ff88-4838-9865-841fba76e96f kind: Event lastTimestamp: "2026-04-23T18:24:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:21Z" name: error-404-isvc-7efef-predictor.18a90f8e32c07dba namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 567e3c50-9b06-4f19-894f-b2e9a3982ac0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24696" uid: 92b46f90-ff88-4838-9865-841fba76e96f kind: Event lastTimestamp: "2026-04-23T18:24:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:51Z" name: error-404-isvc-7efef-predictor.18a90f91b1953b43 namespace: kserve-ci-e2e-test resourceVersion: "25280" uid: 1d4b7dbb-ef38-4908-8999-777a16839063 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24696" uid: 92b46f90-ff88-4838-9865-841fba76e96f kind: Event lastTimestamp: "2026-04-23T18:24:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:51Z" name: error-404-isvc-7efef-predictor.18a90f91b20864bf namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 2fe528d5-c598-461d-ae2d-8eac82278bcd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "24491" uid: 4d62dae7-5b5c-4502-bdc0-aaf6de98ac92 kind: Event lastTimestamp: "2026-04-23T18:24:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:25Z" name: error-404-isvc-7efef.18a90f8ab3aeba62 namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: d77f5c8b-3e35-4823-9d54-596b2db197ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "24491" uid: 4d62dae7-5b5c-4502-bdc0-aaf6de98ac92 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-7efef": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7efef": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-7efef.18a90f8ab4472214 namespace: kserve-ci-e2e-test resourceVersion: "24509" uid: 7398b18e-d3fd-4893-ae60-65ffd883615c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "24491" uid: 4d62dae7-5b5c-4502-bdc0-aaf6de98ac92 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7efef": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-7efef.18a90f8ab4476766 namespace: kserve-ci-e2e-test resourceVersion: "24511" uid: 73bbe8fc-0537-4537-b6ee-b659ded4b1db reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: 4d62dae7-5b5c-4502-bdc0-aaf6de98ac92 kind: Event lastTimestamp: "2026-04-23T18:24:25Z" message: InferenceService [error-404-isvc-7efef] is Ready metadata: creationTimestamp: "2026-04-23T18:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:25Z" name: error-404-isvc-7efef.18a90f960d13e403 namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: e6846eb3-743d-449d-b37b-7966173f9a7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:15.922803Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24160" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f85f100ee83 namespace: kserve-ci-e2e-test resourceVersion: "24164" uid: 23660828-03c0-495f-96cd-8b6ffc74ee05 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24171" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f860a820513 namespace: kserve-ci-e2e-test resourceVersion: "24183" uid: 8b9b7a4d-6d1c-4d67-9552-db75cae80356 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f860becc823 namespace: kserve-ci-e2e-test resourceVersion: "24185" uid: 1c4f4a33-2fc3-4de8-b508-450f74a9914b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f860f9968d8 namespace: kserve-ci-e2e-test resourceVersion: "24186" uid: 4f7cf9f0-251b-434e-bee6-14f0776932db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f861014bdd7 namespace: kserve-ci-e2e-test resourceVersion: "24187" uid: 26e46102-6153-4747-81ac-52754a147fd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f861021bb97 namespace: kserve-ci-e2e-test resourceVersion: "24188" uid: 491cb155-bc51-4c3b-bb4d-55e5428af7af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f8613e3e007 namespace: kserve-ci-e2e-test resourceVersion: "24190" uid: ab1ebb29-703a-4954-967e-50971adff132 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f8614594676 namespace: kserve-ci-e2e-test resourceVersion: "24191" uid: 40a1c59c-3530-4f1b-979e-cf9e7472503a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:23:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:53Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a90f866900cde3 namespace: kserve-ci-e2e-test resourceVersion: "24715" uid: 21e98417-1a01-478f-9c8e-3c5c89eb5bc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:40:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:35Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a91077edad677b namespace: kserve-ci-e2e-test resourceVersion: "32139" uid: 677bccbf-93a4-4448-8b0b-040a7136908b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:40:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:35Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a91077edadba4a namespace: kserve-ci-e2e-test resourceVersion: "32140" uid: cd8b75e1-8713-4d88-bd1d-cb550f966212 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:40:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: eb47bb72-d086-43d1-9dce-8f0a51cbcc01 kind: Event lastTimestamp: "2026-04-23T18:40:38Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:40:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:38Z" name: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt.18a91078c998369f namespace: kserve-ci-e2e-test resourceVersion: "32205" uid: 7e671f79-2456-4cc1-b04e-2e460c59ef74 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-87aee-predictor-54b57bf5ff namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: ff085664-ebd1-44b9-a7ce-ba60cd6a70bc kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'Created pod: error-404-isvc-87aee-predictor-54b57bf5ff-2g5nt' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-87aee-predictor-54b57bf5ff.18a90f85f061ae39 namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: bcc3632e-ba77-4e82-9464-7accb1021c64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24151" uid: 37a6ec56-cb5b-4742-949c-817f302782ae kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Scaled up replica set error-404-isvc-87aee-predictor-54b57bf5ff from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-87aee-predictor.18a90f85ee8d9cf1 namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: bcd3e17d-72d4-44f3-af95-5d9c063aef40 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 88dd7e25-e13e-4f40-89dc-adb32f29a4bb kind: Event lastTimestamp: "2026-04-23T18:23:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:45Z" name: error-404-isvc-87aee-predictor.18a90f896d29869e namespace: kserve-ci-e2e-test resourceVersion: "24647" uid: 3920d957-b158-49b4-84e9-63ab322fbd2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 88dd7e25-e13e-4f40-89dc-adb32f29a4bb kind: Event lastTimestamp: "2026-04-23T18:23:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:45Z" name: error-404-isvc-87aee-predictor.18a90f896da5b18b namespace: kserve-ci-e2e-test resourceVersion: "24648" uid: 435d1c0e-e8a1-4fa2-85b6-d62865893e91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:24:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24375" uid: 88dd7e25-e13e-4f40-89dc-adb32f29a4bb kind: Event lastTimestamp: "2026-04-23T18:24:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:45Z" name: error-404-isvc-87aee-predictor.18a90f906b42fedc namespace: kserve-ci-e2e-test resourceVersion: "25237" uid: 5e2d99ca-35ab-4f3e-ad37-44be82708d56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:24:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24375" uid: 88dd7e25-e13e-4f40-89dc-adb32f29a4bb kind: Event lastTimestamp: "2026-04-23T18:24:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:45Z" name: error-404-isvc-87aee-predictor.18a90f906ba6fe09 namespace: kserve-ci-e2e-test resourceVersion: "25238" uid: 591f3201-0520-4106-a992-9560809188c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 2a2fbb57-ca23-484a-8ee1-9fa4af9cf33b kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-87aee.18a90f85ee67d47a namespace: kserve-ci-e2e-test resourceVersion: "24799" uid: 2f265184-e2dd-40bd-a028-ac8672822480 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 2a2fbb57-ca23-484a-8ee1-9fa4af9cf33b kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-87aee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-87aee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-87aee.18a90f85eec8413d namespace: kserve-ci-e2e-test resourceVersion: "24157" uid: 885d9e9a-5d64-44ee-a927-bfc234ebf6af reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 2a2fbb57-ca23-484a-8ee1-9fa4af9cf33b kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-87aee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-87aee.18a90f85eec89bab namespace: kserve-ci-e2e-test resourceVersion: "24158" uid: 4aee591c-cbd6-413f-a301-a41700eb62d2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24797" uid: 2a2fbb57-ca23-484a-8ee1-9fa4af9cf33b kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: InferenceService [error-404-isvc-87aee] is Ready metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-87aee.18a90f91242d4cd8 namespace: kserve-ci-e2e-test resourceVersion: "24798" uid: a882b145-dabd-42a2-b940-58195dc12fa4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:22:26.115171Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23628" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a93f3-predictor-6ff446966c-c2xqq to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a583cdb71 namespace: kserve-ci-e2e-test resourceVersion: "23632" uid: a5a6ba08-5840-41a8-9ace-b1b4c501111d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23638" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a71d9ec4d namespace: kserve-ci-e2e-test resourceVersion: "23651" uid: c97e51a8-d49a-4e66-b1f6-75e455475340 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a7350f688 namespace: kserve-ci-e2e-test resourceVersion: "23653" uid: 265cb0f7-72a0-4f15-a21c-cca8650a06a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a764c865c namespace: kserve-ci-e2e-test resourceVersion: "23654" uid: 64447cb1-058d-4f8e-a808-f538adee89be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a76c38503 namespace: kserve-ci-e2e-test resourceVersion: "23655" uid: 92c8be9b-a627-484c-bf40-1aa69e92c796 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a76cf7010 namespace: kserve-ci-e2e-test resourceVersion: "23656" uid: 97b4d4c2-df2a-4fb1-969f-6108cc4db17b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a797fe56a namespace: kserve-ci-e2e-test resourceVersion: "23658" uid: 73f57130-4107-4d21-b719-6a29e838a553 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7a79f5028f namespace: kserve-ci-e2e-test resourceVersion: "23659" uid: 335bf751-e424-4142-a5a4-57f41440bc4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:23:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:03Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f7ab9c98124 namespace: kserve-ci-e2e-test resourceVersion: "23964" uid: 1ea8b48f-1413-474f-9ce0-b9fe2d2b3b8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f8aaacebea1 namespace: kserve-ci-e2e-test resourceVersion: "24455" uid: 3f510dd5-d373-4bda-9321-012bbe4706c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f8aaacf538a namespace: kserve-ci-e2e-test resourceVersion: "24457" uid: b6c6488a-62b3-4155-8d93-eaf4bae86d52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: 645360c6-749d-41eb-9e30-9ba98e4a59c6 kind: Event lastTimestamp: "2026-04-23T18:23:38Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:38Z" name: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq.18a90f8b41e1c6b9 namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 4bc29718-2c19-473b-b1d1-52e250a50302 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a93f3-predictor-6ff446966c namespace: kserve-ci-e2e-test resourceVersion: "23619" uid: 7cbea4f0-fbf5-4ca3-9ad6-76822624c425 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'Created pod: error-404-isvc-a93f3-predictor-6ff446966c-c2xqq' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor-6ff446966c.18a90f7a57cf7aae namespace: kserve-ci-e2e-test resourceVersion: "23629" uid: 98a8c4fa-b309-4fa9-8c02-efee7cdbe09a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23615" uid: 9db36fd6-7849-477f-8e81-3c13027313b3 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: Scaled up replica set error-404-isvc-a93f3-predictor-6ff446966c from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3-predictor.18a90f7a55fe3d1e namespace: kserve-ci-e2e-test resourceVersion: "23620" uid: 11976fcb-5c12-4ad5-802f-d58431119740 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23618" uid: c0d54682-d68a-454f-9ddd-1136f1a12e26 kind: Event lastTimestamp: "2026-04-23T18:22:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:56Z" name: error-404-isvc-a93f3-predictor.18a90f7dd4b192a0 namespace: kserve-ci-e2e-test resourceVersion: "23908" uid: 62c8086f-98ab-455c-bde9-cb4093607c48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23618" uid: c0d54682-d68a-454f-9ddd-1136f1a12e26 kind: Event lastTimestamp: "2026-04-23T18:22:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:56Z" name: error-404-isvc-a93f3-predictor.18a90f7dd52669da namespace: kserve-ci-e2e-test resourceVersion: "23909" uid: aedf88fc-129b-4906-bc15-6051fa0bafd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: c0d54682-d68a-454f-9ddd-1136f1a12e26 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:26Z" name: error-404-isvc-a93f3-predictor.18a90f84d2e8a007 namespace: kserve-ci-e2e-test resourceVersion: "24324" uid: 08e123ac-2e62-4f23-8e41-767ca019d348 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23794" uid: c0d54682-d68a-454f-9ddd-1136f1a12e26 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:26Z" name: error-404-isvc-a93f3-predictor.18a90f84d3530bf7 namespace: kserve-ci-e2e-test resourceVersion: "24325" uid: a405325b-f1a1-407e-b4c9-2db5606ca1fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "23587" uid: f94495e5-0ece-4b16-b438-6bede2eb668c kind: Event lastTimestamp: "2026-04-23T18:23:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:13Z" name: error-404-isvc-a93f3.18a90f7a56021f12 namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 9549bc7b-1988-4693-a669-a7ad1d014d03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "23587" uid: f94495e5-0ece-4b16-b438-6bede2eb668c kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-a93f3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a93f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3.18a90f7a56b3b72f namespace: kserve-ci-e2e-test resourceVersion: "23624" uid: bb33aa2f-ddb3-446c-a4e8-703b04fabd44 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "23587" uid: f94495e5-0ece-4b16-b438-6bede2eb668c kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a93f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:26Z" name: error-404-isvc-a93f3.18a90f7a56b402b7 namespace: kserve-ci-e2e-test resourceVersion: "23626" uid: d0bccef7-c3fe-4690-a9bf-c539f2207bff reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "24042" uid: f94495e5-0ece-4b16-b438-6bede2eb668c kind: Event lastTimestamp: "2026-04-23T18:23:13Z" message: InferenceService [error-404-isvc-a93f3] is Ready metadata: creationTimestamp: "2026-04-23T18:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:13Z" name: error-404-isvc-a93f3.18a90f8574ac5a1b namespace: kserve-ci-e2e-test resourceVersion: "24043" uid: 0fd3733f-de18-41dc-b509-7b614782f089 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:40.827542Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d864a-predictor-f8b8f6449-xvnkn to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef212605c32 namespace: kserve-ci-e2e-test resourceVersion: "18830" uid: 14cc6326-b5a9-4c54-b945-b3c3e72b28e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d864a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:41Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef220832c00 namespace: kserve-ci-e2e-test resourceVersion: "18839" uid: fc566995-e2a1-4f08-afc9-bc7d4c3632ee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:41Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef2500930ec namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: ef2a229c-97b3-4253-a318-ad232226d1c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:41Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef25171ecf0 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 15c5f48f-1f46-44a0-9d8d-7b219fb7cac0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:41Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef2547cd80c namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: 0c7a9ee4-94ba-484a-8bf1-da72ebad4a5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:41Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef254f46f20 namespace: kserve-ci-e2e-test resourceVersion: "18857" uid: 7563e3ec-6eec-4895-a96a-46df8eb849a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:41Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef25500e674 namespace: kserve-ci-e2e-test resourceVersion: "18859" uid: 74596b2b-814e-4f6f-bae2-bcc062985cf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef25921e300 namespace: kserve-ci-e2e-test resourceVersion: "18860" uid: 2590e644-2dd4-439d-bc48-e5c98c4cbe9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef2599bc4b3 namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 546e6e5a-c390-435e-a750-08d135162b14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90ef2bbeecd24 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 6434366e-268e-4e95-88a6-5ec80f84258c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90f04be2159a4 namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: ac474c68-1748-4d66-8925-3d8bbf64a895 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90f04be21ff43 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 2fa21601-558b-413e-bb8d-6c9f7bde7638 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 514bdb74-73a0-4a62-a268-22a2bb73d08c kind: Event lastTimestamp: "2026-04-23T18:14:04Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:04Z" name: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn.18a90f05983f120a namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 51b81886-edba-4818-9dba-289068631132 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d864a-predictor-f8b8f6449 namespace: kserve-ci-e2e-test resourceVersion: "18819" uid: e66d5adb-4d29-4b27-9b53-d691e0f7b746 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: 'Created pod: error-404-isvc-d864a-predictor-f8b8f6449-xvnkn' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-d864a-predictor-f8b8f6449.18a90ef211f38ccb namespace: kserve-ci-e2e-test resourceVersion: "18827" uid: 9f67ccf1-b5f0-4899-b925-2400628e6d84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: c449e5bf-afc1-4873-87bb-0649d5cc856d kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Scaled up replica set error-404-isvc-d864a-predictor-f8b8f6449 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-d864a-predictor.18a90ef210741fdc namespace: kserve-ci-e2e-test resourceVersion: "18821" uid: 56a56b90-7a5d-4648-967a-ffd97495e7aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18818" uid: 47e8ff44-8ad4-45dc-b224-276a50dcd451 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:10Z" name: error-404-isvc-d864a-predictor.18a90ef58f098ded namespace: kserve-ci-e2e-test resourceVersion: "19189" uid: fc998ba0-79a1-4369-9c26-487db023b7d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18818" uid: 47e8ff44-8ad4-45dc-b224-276a50dcd451 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:10Z" name: error-404-isvc-d864a-predictor.18a90ef58f7a318a namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: 87ec4fb3-c304-4957-a9a9-b4e9058a3c75 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 47e8ff44-8ad4-45dc-b224-276a50dcd451 kind: Event lastTimestamp: "2026-04-23T18:13:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:55Z" name: error-404-isvc-d864a-predictor.18a90efc8d17c2fc namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: eac7bb81-5814-486f-bda9-415f458304d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 47e8ff44-8ad4-45dc-b224-276a50dcd451 kind: Event lastTimestamp: "2026-04-23T18:13:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:55Z" name: error-404-isvc-d864a-predictor.18a90efc8d8eaacf namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: c9b5fd86-61c9-40bd-b13e-dc39d4d399a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: 9002787c-e60f-4fdb-ad0e-b410610884a9 kind: Event lastTimestamp: "2026-04-23T18:13:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:29Z" name: error-404-isvc-d864a.18a90ef21047e108 namespace: kserve-ci-e2e-test resourceVersion: "19492" uid: 1471703c-f04d-4ab0-b1a4-df0c55856ead reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: 9002787c-e60f-4fdb-ad0e-b410610884a9 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-d864a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d864a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-d864a.18a90ef210b8c8f5 namespace: kserve-ci-e2e-test resourceVersion: "18823" uid: 250c7bb9-41c5-4524-a418-0c6458c5403e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: 9002787c-e60f-4fdb-ad0e-b410610884a9 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d864a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:40Z" name: error-404-isvc-d864a.18a90ef210b922bb namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 7608c4d3-2029-48ed-9474-724dc39f7ede reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 9002787c-e60f-4fdb-ad0e-b410610884a9 kind: Event lastTimestamp: "2026-04-23T18:13:29Z" message: InferenceService [error-404-isvc-d864a] is Ready metadata: creationTimestamp: "2026-04-23T18:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:29Z" name: error-404-isvc-d864a.18a90efd777ccc52 namespace: kserve-ci-e2e-test resourceVersion: "19491" uid: 5d1ea1c7-1f8a-49bc-a3a3-756bf25e471d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:14:01.071727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f04c14d6775 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: b0711acd-504c-4570-9523-3581e81b356a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e1fac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f04d266b114 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: c1bbeb07-bc75-4c42-bdce-8c967d5027a8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f04fe7a4e97 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 091e3d73-4493-4654-904d-92bec6e848c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f050020338b namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: c2cd365e-21c2-4e6b-a846-cc7b20eeeb1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f05031b1245 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 39c69673-8502-4ea0-812b-3849060f0a2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f05039169cc namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: b3eed47a-70c2-4a58-9b7e-36d4afcf3b99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f05039d7276 namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 10f4c901-95bf-485d-ad67-9fe449f2a05c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f05070a3db7 namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 6e7f5b89-5f06-4928-a535-377bdeab0d71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:02Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f05079300fa namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: dc06fe89-f075-40b9-adb5-dff8ae05ac65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:14:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:14:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:14:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:39Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f056ef6583e namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 2fe8cc05-9449-459e-9d09-c50b144b45a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f85ecdc986d namespace: kserve-ci-e2e-test resourceVersion: "24146" uid: 14ffb58e-2a73-42af-a098-4a494cc681a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 4368b22b-f54e-44eb-8dcf-c3117cceb717 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:15Z" name: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj.18a90f85ecdcdb72 namespace: kserve-ci-e2e-test resourceVersion: "24148" uid: 933b8449-8eec-46c3-83ce-041d09d7c33b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e1fac-predictor-779fdf88dc namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: d8fd4cd4-0a00-41a6-887d-1e8a04059a13 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: 'Created pod: error-404-isvc-e1fac-predictor-779fdf88dc-7mrqj' metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-e1fac-predictor-779fdf88dc.18a90f04c0da395a namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 509c4e3d-95f4-4680-83d0-06fdf8167a0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 8c5d2cfc-9e67-4c60-9b89-ddfd65e9431f kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Scaled up replica set error-404-isvc-e1fac-predictor-779fdf88dc from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-e1fac-predictor.18a90f04bf56dcae namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 460d3f7d-8ba0-405d-9ad8-18344eb9f12b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: b1328e55-b8fe-4011-aa9e-fa058e164ba5 kind: Event lastTimestamp: "2026-04-23T18:14:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:16Z" name: error-404-isvc-e1fac-predictor.18a90f083e32931b namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 011c8a6b-025a-4122-8fae-7c2fa6974a37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: b1328e55-b8fe-4011-aa9e-fa058e164ba5 kind: Event lastTimestamp: "2026-04-23T18:14:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:16Z" name: error-404-isvc-e1fac-predictor.18a90f083eaff20b namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: dfefcf32-d5a9-49da-8152-5864248b0a62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: b1328e55-b8fe-4011-aa9e-fa058e164ba5 kind: Event lastTimestamp: "2026-04-23T18:15:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:16Z" name: error-404-isvc-e1fac-predictor.18a90f0bbd9f2d60 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 149b9d46-53e4-479a-bbc2-2030c375f529 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: b1328e55-b8fe-4011-aa9e-fa058e164ba5 kind: Event lastTimestamp: "2026-04-23T18:15:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:16Z" name: error-404-isvc-e1fac-predictor.18a90f0bbe164837 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 03865aa0-383e-4d7b-bd94-084d93bae93b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: d2d27973-2666-4874-aeb4-de5d60e11f72 kind: Event lastTimestamp: "2026-04-23T18:14:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:50Z" name: error-404-isvc-e1fac.18a90f04bf30d5c7 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 9ae7582e-1df3-4469-a73c-c69ca00093a3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: d2d27973-2666-4874-aeb4-de5d60e11f72 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-e1fac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e1fac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-e1fac.18a90f04bf915cd7 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: c713a30b-7ffc-4e7d-933b-7afe83870b90 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: d2d27973-2666-4874-aeb4-de5d60e11f72 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e1fac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:01Z" name: error-404-isvc-e1fac.18a90f04bf91b5e2 namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: 047c8471-5f1d-4fcc-85a6-7dc995d06ec3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: d2d27973-2666-4874-aeb4-de5d60e11f72 kind: Event lastTimestamp: "2026-04-23T18:14:50Z" message: InferenceService [error-404-isvc-e1fac] is Ready metadata: creationTimestamp: "2026-04-23T18:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:50Z" name: error-404-isvc-e1fac.18a90f1029e48534 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 4dd5d4da-37fb-41e5-9cc7-1e4926ba101b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:01:16.005166Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12903" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:01:16Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e529fc82ac1 namespace: kserve-ci-e2e-test resourceVersion: "12907" uid: 7eda9a70-b471-4747-9760-4c2931aabcea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12912" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:01:16Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e52ba73fb8a namespace: kserve-ci-e2e-test resourceVersion: "12936" uid: 006095c3-7d66-40f2-86c4-44a2b296de95 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:16Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e52bbec1eb3 namespace: kserve-ci-e2e-test resourceVersion: "12938" uid: 99f620f9-a2f0-45dd-a4d6-f7a4785f4269 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1417" in 13.73s (13.73s including waiting). Image size: 1336611437 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e55ee5477fa namespace: kserve-ci-e2e-test resourceVersion: "13099" uid: 759499f6-37c6-4b38-90b4-85d6a4933457 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e55f19c551e namespace: kserve-ci-e2e-test resourceVersion: "13104" uid: 678c14da-dda7-470b-bcb2-26da2f49b68b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e55f2120707 namespace: kserve-ci-e2e-test resourceVersion: "13105" uid: 1f2c8d81-8583-4564-b3b4-80ad93359c82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e55f2255d11 namespace: kserve-ci-e2e-test resourceVersion: "13108" uid: 057db580-1c92-400e-ab36-f9156b0dde14 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:32Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.587s (2.587s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:32Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e568c5b40ae namespace: kserve-ci-e2e-test resourceVersion: "13154" uid: f385a2a6-2428-465a-beca-193c9942ac2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:32Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e5691e6dafe namespace: kserve-ci-e2e-test resourceVersion: "13155" uid: 289a34fe-29b7-4c6e-b72e-555bf15418dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:01:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:32Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e569304839f namespace: kserve-ci-e2e-test resourceVersion: "13156" uid: 6fb4bd1b-3907-4c4d-91dd-9267590481f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:01:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:02:09Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:09Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e56a4f9489d namespace: kserve-ci-e2e-test resourceVersion: "13486" uid: 4504142d-fabb-433f-baeb-d9c2f701db97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e678d31869c namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: 1661f1d6-4370-4b28-a76b-3b776075c419 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6 namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 6b172450-c872-4357-bc94-d89fe33e4343 kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6.18a90e678d323c9c namespace: kserve-ci-e2e-test resourceVersion: "13917" uid: 06b630b5-6592-4993-94fd-015639823d7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e7d75-predictor-7d6bff459f namespace: kserve-ci-e2e-test resourceVersion: "12884" uid: 757f01ec-d0f6-4cf3-b6f0-efc95f7a6948 kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: 'Created pod: error-404-isvc-e7d75-predictor-7d6bff459f-l5fn6' metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: error-404-isvc-e7d75-predictor-7d6bff459f.18a90e529f5d06ea namespace: kserve-ci-e2e-test resourceVersion: "12904" uid: 01fc91db-25b4-4e97-9789-a1971f2110ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "12882" uid: 80e56089-105f-4693-b9cd-636d9c8ef46a kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: Scaled up replica set error-404-isvc-e7d75-predictor-7d6bff459f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: error-404-isvc-e7d75-predictor.18a90e529a318721 namespace: kserve-ci-e2e-test resourceVersion: "12887" uid: d60a6b11-46ff-42cc-a2f0-fd138a9c12c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "12883" uid: 223893ed-278c-4cea-ab5d-231b1fdb6747 kind: Event lastTimestamp: "2026-04-23T18:02:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:00Z" name: error-404-isvc-e7d75-predictor.18a90e5618ba106c namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 5908acc9-fdfa-4fc6-be30-6817a4ebfdc1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "12883" uid: 223893ed-278c-4cea-ab5d-231b1fdb6747 kind: Event lastTimestamp: "2026-04-23T18:02:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:00Z" name: error-404-isvc-e7d75-predictor.18a90e56192d404a namespace: kserve-ci-e2e-test resourceVersion: "13428" uid: e428323d-75f6-424f-8009-70ba7f3326da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "13126" uid: 223893ed-278c-4cea-ab5d-231b1fdb6747 kind: Event lastTimestamp: "2026-04-23T18:02:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:30Z" name: error-404-isvc-e7d75-predictor.18a90e6096041cdd namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 7ce333ea-3290-4fb9-80c3-c349f5846e7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "13126" uid: 223893ed-278c-4cea-ab5d-231b1fdb6747 kind: Event lastTimestamp: "2026-04-23T18:02:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:30Z" name: error-404-isvc-e7d75-predictor.18a90e60967abd4a namespace: kserve-ci-e2e-test resourceVersion: "13686" uid: 83195026-81df-4d19-91fd-e4a7f395ff66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7d75 namespace: kserve-ci-e2e-test resourceVersion: "12875" uid: 1c12b1c6-9a84-42e3-8eb8-7fe4a517d82f kind: Event lastTimestamp: "2026-04-23T18:02:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:19Z" name: error-404-isvc-e7d75.18a90e529a051031 namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: 6ed14f11-ad05-479d-8c92-954a00488a24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7d75 namespace: kserve-ci-e2e-test resourceVersion: "13578" uid: 1c12b1c6-9a84-42e3-8eb8-7fe4a517d82f kind: Event lastTimestamp: "2026-04-23T18:02:19Z" message: InferenceService [error-404-isvc-e7d75] is Ready metadata: creationTimestamp: "2026-04-23T18:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:19Z" name: error-404-isvc-e7d75.18a90e6160531d37 namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: 0712b665-75c5-4dc2-a40d-c6f1dd475f50 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:01:15.620023Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12820" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m to ip-10-0-136-172.ec2.internal metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:01:15Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e5288d34ec3 namespace: kserve-ci-e2e-test resourceVersion: "12827" uid: 7f8a52f4-ea9f-499a-8756-821b4a550eb0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12831" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:01:16Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e52a3ebecfd namespace: kserve-ci-e2e-test resourceVersion: "12913" uid: 9bd0d644-db5b-44a4-96db-a0a4eb1ba53e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:16Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e52a5771079 namespace: kserve-ci-e2e-test resourceVersion: "12916" uid: 1abdd3e2-a47f-45d3-9aa3-73d3599603a5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" in 14.036s (14.036s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e55ea160384 namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 8a133025-6fe9-4799-b04c-ea0afe9ff211 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e55ef09f830 namespace: kserve-ci-e2e-test resourceVersion: "13100" uid: a132b7e6-f788-4155-a40f-de0e24e7a2bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:30Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e55ef96144f namespace: kserve-ci-e2e-test resourceVersion: "13101" uid: 707071cc-ee36-4715-846e-8a1266413841 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:35Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" metadata: creationTimestamp: "2026-04-23T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:35Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e571c93420f namespace: kserve-ci-e2e-test resourceVersion: "13185" uid: 58c39f3b-5c43-4198-8aa8-a9620f2a4c2d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" in 6.521s (6.521s including waiting). Image size: 1562812853 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58a145be71 namespace: kserve-ci-e2e-test resourceVersion: "13244" uid: 44306877-7e78-474a-b129-1453b1d750af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58a4fad177 namespace: kserve-ci-e2e-test resourceVersion: "13246" uid: eb84b590-5bf9-4097-9c73-4682574a5ed0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58a565ddfd namespace: kserve-ci-e2e-test resourceVersion: "13248" uid: aa3fbbe6-4b69-45db-9e9c-183118fb506e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58a58326f1 namespace: kserve-ci-e2e-test resourceVersion: "13250" uid: 235c7608-6b4b-4ac3-8d03-aaaa8ad9e75d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58a936956d namespace: kserve-ci-e2e-test resourceVersion: "13254" uid: 35105742-823a-492c-b6d7-dbbc9bfd2a58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58a999ed97 namespace: kserve-ci-e2e-test resourceVersion: "13255" uid: 9a2c5ac1-1393-483c-a66d-209d68d7eab8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:03:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:29Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e58fb85015d namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: da1de74d-6b6f-4ae0-b622-f3a01efc88a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e70db9543b3 namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: f901e76e-14d0-4312-aff9-23fc526f176a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e70db95b8b8 namespace: kserve-ci-e2e-test resourceVersion: "14467" uid: dfe245ad-9336-46a6-ae2b-837d8ab1740f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m namespace: kserve-ci-e2e-test resourceVersion: "12825" uid: 0384d20a-86ac-4a4d-85e3-6e6f1f775895 kind: Event lastTimestamp: "2026-04-23T18:03:29Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:29Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m.18a90e71a9cd43b3 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 61eff24a-883c-4f07-82e4-7bb6d93333fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-172.ec2.internal source: component: kubelet host: ip-10-0-136-172.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84c4f457f6 namespace: kserve-ci-e2e-test resourceVersion: "12809" uid: 4f963536-d62f-4fcf-9349-f186b11f96be kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84c4f457f6-x2b4m' metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: isvc-sklearn-graph-1-predictor-84c4f457f6.18a90e528867d330 namespace: kserve-ci-e2e-test resourceVersion: "12824" uid: c59712e4-8dbd-4dcc-a8b2-493ec76c5b75 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12807" uid: 0c060be5-86ba-46dc-bdf7-6a8bdd23dc8f kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84c4f457f6 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: isvc-sklearn-graph-1-predictor.18a90e5281ff32db namespace: kserve-ci-e2e-test resourceVersion: "12810" uid: 1fb10840-bfba-40f1-89c3-4a97edbd1037 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12818" uid: e4046fa3-fd39-4331-84f6-0dba48cc583c kind: Event lastTimestamp: "2026-04-23T18:02:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:15Z" name: isvc-sklearn-graph-1-predictor.18a90e56074bacb0 namespace: kserve-ci-e2e-test resourceVersion: "13539" uid: 926debae-4b43-4f3b-9c28-5aae0b51d732 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12818" uid: e4046fa3-fd39-4331-84f6-0dba48cc583c kind: Event lastTimestamp: "2026-04-23T18:02:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:15Z" name: isvc-sklearn-graph-1-predictor.18a90e5607b913be namespace: kserve-ci-e2e-test resourceVersion: "13540" uid: ce671713-a9f5-4468-b61b-29e54e039b35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: e4046fa3-fd39-4331-84f6-0dba48cc583c kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:15Z" name: isvc-sklearn-graph-1-predictor.18a90e5d061ca852 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 735c665c-d870-4f11-bf12-52ee250cc712 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: e4046fa3-fd39-4331-84f6-0dba48cc583c kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:15Z" name: isvc-sklearn-graph-1-predictor.18a90e5d066e13a1 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: c44a1b79-1551-4237-a14e-e89a56998c54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "12795" uid: fa4dd7c1-dd9b-4fb1-b01a-7efe802d8735 kind: Event lastTimestamp: "2026-04-23T18:02:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:49Z" name: isvc-sklearn-graph-1.18a90e5288204c5f namespace: kserve-ci-e2e-test resourceVersion: "14022" uid: dd5f3baf-2790-4ebb-b666-9e06c3b3e8c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: fa4dd7c1-dd9b-4fb1-b01a-7efe802d8735 kind: Event lastTimestamp: "2026-04-23T18:02:49Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-23T18:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:49Z" name: isvc-sklearn-graph-1.18a90e685ea3b96c namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: b7ebb6e2-638c-44fb-819e-8f09f6317b25 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:01:16.726268Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12944" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:01:16Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e52cac354a2 namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 20b99c8c-8e6b-429c-90ef-513c308501e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12954" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:17Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:01:17Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e52ea0924a3 namespace: kserve-ci-e2e-test resourceVersion: "12966" uid: f58be297-20fc-40a4-b754-9b0b5a769d43 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:17Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" metadata: creationTimestamp: "2026-04-23T18:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:17Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e52eba9a8c0 namespace: kserve-ci-e2e-test resourceVersion: "12969" uid: af2b1ccc-e65a-425b-b189-b62d93a2d8b8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" in 11.894s (11.894s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e55b0a83a64 namespace: kserve-ci-e2e-test resourceVersion: "13076" uid: b051bb36-6d2c-4542-b609-337b9a0a5a3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e55b5369229 namespace: kserve-ci-e2e-test resourceVersion: "13078" uid: 7bfc3e79-5d6f-49c9-a809-383cc23aa5aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e55b5adf851 namespace: kserve-ci-e2e-test resourceVersion: "13079" uid: cfc98c81-8aea-488a-8ca8-32cc86afe02e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:33Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" metadata: creationTimestamp: "2026-04-23T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:33Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e56b054a7bc namespace: kserve-ci-e2e-test resourceVersion: "13164" uid: 55f8a7ae-2493-4a64-a0cc-dff664b9b293 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" in 8.328s (8.328s including waiting). Image size: 1562812853 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e58a0c53a5c namespace: kserve-ci-e2e-test resourceVersion: "13243" uid: fe884187-cbaf-4549-a482-e20de48eee3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e58a489a2d1 namespace: kserve-ci-e2e-test resourceVersion: "13245" uid: 0ea680a6-0cb4-49e8-a24c-6bf745e9d1f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e58a50aab00 namespace: kserve-ci-e2e-test resourceVersion: "13247" uid: 99ac3c6d-1e38-4b89-9dc1-32f649cfcf4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e58a523cd0a namespace: kserve-ci-e2e-test resourceVersion: "13249" uid: df849e28-a551-4ba6-9a5a-f80f959cdd71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e58a89c827d namespace: kserve-ci-e2e-test resourceVersion: "13252" uid: 686a04cb-7462-4509-9bf3-ed8b3011cc43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e58a911e9bd namespace: kserve-ci-e2e-test resourceVersion: "13253" uid: 9e9e53c0-b0fa-4b9a-9ed4-650486f4fc53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:03:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:29Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e59068c7770 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 0b0c7eeb-8a3a-4ad8-8ee5-a136f4b37f4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e70d84f4ea1 namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 53048c59-243c-4ade-b989-759f1591b846 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e70d8586e29 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 905b4c01-6af5-48af-8ba3-db44d1f6406c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 96fb8faf-c4f3-4431-a460-c27f10b43e34 kind: Event lastTimestamp: "2026-04-23T18:03:29Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:29Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92.18a90e71b4c2dc2b namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 3860f833-9b1a-427c-9350-146730d17db1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-57f87c4ccb namespace: kserve-ci-e2e-test resourceVersion: "12929" uid: f3232594-e972-47ea-ae92-02e6c2447d6d kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-57f87c4ccb-wbq92' metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:16Z" name: isvc-sklearn-graph-2-predictor-57f87c4ccb.18a90e52ca555eb0 namespace: kserve-ci-e2e-test resourceVersion: "12945" uid: 0d5220d5-59a0-46bd-8472-8aa2b790d285 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12927" uid: 7d78d928-51e2-4ac7-ba40-8ac21fe01b2b kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-57f87c4ccb from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:16Z" name: isvc-sklearn-graph-2-predictor.18a90e52b3da43b9 namespace: kserve-ci-e2e-test resourceVersion: "12930" uid: 80fb4f50-36cd-4cc0-856b-4df8fbe9f389 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:01:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12928" uid: d5e1b9f9-7213-49ca-a943-f4d07f7f6e7f kind: Event lastTimestamp: "2026-04-23T18:02:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:16Z" name: isvc-sklearn-graph-2-predictor.18a90e56326927e3 namespace: kserve-ci-e2e-test resourceVersion: "13553" uid: fddb1814-22ff-4fdc-be6c-eb4c3c83b1b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:01:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12928" uid: d5e1b9f9-7213-49ca-a943-f4d07f7f6e7f kind: Event lastTimestamp: "2026-04-23T18:02:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:16Z" name: isvc-sklearn-graph-2-predictor.18a90e5632dd0c8a namespace: kserve-ci-e2e-test resourceVersion: "13554" uid: 4faac716-4ed7-4665-9f04-f34e5385dcc9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:02:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13133" uid: d5e1b9f9-7213-49ca-a943-f4d07f7f6e7f kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:16Z" name: isvc-sklearn-graph-2-predictor.18a90e642e0013b1 namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: cbd91640-597d-4b99-b975-e4c5123a8ac1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:02:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13133" uid: d5e1b9f9-7213-49ca-a943-f4d07f7f6e7f kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:16Z" name: isvc-sklearn-graph-2-predictor.18a90e642e93949f namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 077b4790-1a8d-42b4-8512-5307e67489c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "12911" uid: 0c946aa3-f4a0-43d9-93da-da03b4fb5e20 kind: Event lastTimestamp: "2026-04-23T18:02:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:49Z" name: isvc-sklearn-graph-2.18a90e52b3b002da namespace: kserve-ci-e2e-test resourceVersion: "14035" uid: bef3496c-2913-40da-9308-5db189b67155 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14033" uid: 0c946aa3-f4a0-43d9-93da-da03b4fb5e20 kind: Event lastTimestamp: "2026-04-23T18:02:49Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-23T18:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:49Z" name: isvc-sklearn-graph-2.18a90e6869f31fb1 namespace: kserve-ci-e2e-test resourceVersion: "14034" uid: 3b3d786d-42b1-4cf1-96b1-26f685fda790 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:01:15.925278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12885" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-rzx5c to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:01:15Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e529b06cefe namespace: kserve-ci-e2e-test resourceVersion: "12893" uid: 1cb912a0-1057-47d9-ab66-cba6fcc891f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:16Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e52a417a156 namespace: kserve-ci-e2e-test resourceVersion: "12914" uid: d9b52dab-2e0f-47aa-b6a1-2e154b481f69 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12900" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:01:16Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e52d78c09c7 namespace: kserve-ci-e2e-test resourceVersion: "12959" uid: 1cf55c61-ffc6-42af-9830-1ee2ed4caa57 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:16Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e52d8fffab4 namespace: kserve-ci-e2e-test resourceVersion: "12961" uid: 73c235a2-0c11-430d-abd9-ea0447d6ee09 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:21Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" in 4.11s (4.11s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:21Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e53ce08dcad namespace: kserve-ci-e2e-test resourceVersion: "13008" uid: bf9c49fd-5f6e-4eee-867f-62445f9f580b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:21Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e53d15f09f4 namespace: kserve-ci-e2e-test resourceVersion: "13010" uid: 503533a9-698d-4244-afdb-7afa187ff1cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:21Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e53d1d2488f namespace: kserve-ci-e2e-test resourceVersion: "13011" uid: 8c4b66a4-95ab-447c-a8e4-df6b9fde133b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:24Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-23T18:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:24Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e54ae8b9e56 namespace: kserve-ci-e2e-test resourceVersion: "13046" uid: d6d58940-dfa6-4639-ba44-e687a10bfd01 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 16.261s (16.261s including waiting). Image size: 1306414326 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e5877ce3747 namespace: kserve-ci-e2e-test resourceVersion: "13234" uid: 26a7114f-d0fe-4c06-afa5-9fb3b27bc7bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e587b18470c namespace: kserve-ci-e2e-test resourceVersion: "13236" uid: 03274797-4309-4898-ba75-e0413089678a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e587b87f1bb namespace: kserve-ci-e2e-test resourceVersion: "13237" uid: a666d996-1bb7-486f-b2ea-1b0bb2034a1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e587b947e52 namespace: kserve-ci-e2e-test resourceVersion: "13238" uid: 64c4ab23-b1cf-46d9-b59e-d9eb82df0fff reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:43Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.137s (2.137s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e58fafab7b9 namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: 89c65fe6-1829-4541-a9ed-327c6277aec0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e58fdf16e80 namespace: kserve-ci-e2e-test resourceVersion: "13269" uid: c9dac23e-c0dd-42fb-8aae-844c400fe22f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:01:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e58fe6200c6 namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 49d42932-df60-4451-a764-0f76c0695a21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:01:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:02:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e595ab86ef4 namespace: kserve-ci-e2e-test resourceVersion: "13682" uid: be5a2d15-682c-4c0f-9f3b-692ee22cf45a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e70d4f8f9a0 namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: a4fc8e44-1a4b-43ef-8908-7885802e54dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e70d4f96339 namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: ca7d5b0a-3bfa-418e-80ff-23ae8a820f9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 9742f92c-6d03-44d2-892a-06e0d31fab64 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: isvc-xgboost-graph-predictor-669d8d6456-rzx5c.18a90e70def0c2cd namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 0c154054-f141-4332-9e26-e0f89e497ecd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "12866" uid: fc12ebe1-cc66-408d-b10b-b39593c9d210 kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-rzx5c' metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a90e529a744f22 namespace: kserve-ci-e2e-test resourceVersion: "12890" uid: cf0d0220-6e41-48d9-bc60-e365dc085fd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12864" uid: 11e811ff-e36c-47d3-90e6-b45a3fd5c45b kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: isvc-xgboost-graph-predictor.18a90e529459cabc namespace: kserve-ci-e2e-test resourceVersion: "12868" uid: 0d69d6f9-16ea-4932-8427-b5bb17417e32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12865" uid: 34ad6a30-2890-40e9-9f82-914b7d6ae818 kind: Event lastTimestamp: "2026-04-23T18:02:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:30Z" name: isvc-xgboost-graph-predictor.18a90e5612f37d95 namespace: kserve-ci-e2e-test resourceVersion: "13680" uid: 586d8c92-68cf-466d-b7d6-5f7ed1a61afb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12865" uid: 34ad6a30-2890-40e9-9f82-914b7d6ae818 kind: Event lastTimestamp: "2026-04-23T18:02:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:30Z" name: isvc-xgboost-graph-predictor.18a90e56135e79fe namespace: kserve-ci-e2e-test resourceVersion: "13681" uid: 62c0e578-30ba-41da-99e1-3494c68cf12c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13123" uid: 34ad6a30-2890-40e9-9f82-914b7d6ae818 kind: Event lastTimestamp: "2026-04-23T18:03:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:00Z" name: isvc-xgboost-graph-predictor.18a90e678d692b1d namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 54ef0c58-d360-4732-a5b9-3879e5877a9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13123" uid: 34ad6a30-2890-40e9-9f82-914b7d6ae818 kind: Event lastTimestamp: "2026-04-23T18:03:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:00Z" name: isvc-xgboost-graph-predictor.18a90e678dde4420 namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: 17eb0860-ef24-465b-9b22-9dce092e2aa8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "12849" uid: 0e7490a1-adc4-41b5-95f2-c80e85ec4619 kind: Event lastTimestamp: "2026-04-23T18:02:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:41Z" name: isvc-xgboost-graph.18a90e52942e4369 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 04dd4e99-bd5c-4b6e-8662-037c1046edaa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 0e7490a1-adc4-41b5-95f2-c80e85ec4619 kind: Event lastTimestamp: "2026-04-23T18:02:41Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-23T18:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:41Z" name: isvc-xgboost-graph.18a90e666b7f431b namespace: kserve-ci-e2e-test resourceVersion: "13806" uid: 7c02c327-16f0-4e4b-8bda-4f877be13aec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:03:15.571032Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5894656ff9-2kqdb to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:03:15Z" name: model-chainer-5894656ff9-2kqdb.18a90e6e76767a4e namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: 87715ac5-a839-436a-b1de-bdb64e95e061 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:15Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:15Z" name: model-chainer-5894656ff9-2kqdb.18a90e6e8153292d namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 1f8ec44f-667f-42b4-b9db-cecef9de6bb2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:16Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:03:16Z" name: model-chainer-5894656ff9-2kqdb.18a90e6eb3000037 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: e188d9ab-6d26-459d-91d3-eb0e9641dca0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:16Z" name: model-chainer-5894656ff9-2kqdb.18a90e6eb448391c namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 1a1056d2-702d-4263-a2f1-4dc145e78926 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-23T18:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:16Z" name: model-chainer-5894656ff9-2kqdb.18a90e6eb7799b8d namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 536c2c2d-56ae-4bee-98e5-eb25821b8373 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:16Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-23T18:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:16Z" name: model-chainer-5894656ff9-2kqdb.18a90e6eb7fef148 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: a621ce7b-23c1-4280-9a19-87aec5c5b47e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:25Z" name: model-chainer-5894656ff9-2kqdb.18a90e70d00421f7 namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 46d6411c-2c80-488c-9554-56ed5920aa50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894656ff9-2kqdb namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 58386c59-d272-43c1-b6bc-ad193ef4461e kind: Event lastTimestamp: "2026-04-23T18:03:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:53Z" name: model-chainer-5894656ff9-2kqdb.18a90e7165808f61 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 92ba02d5-f8bf-48a7-b27b-30e7e9d69c33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5894656ff9 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: 185ccff1-a593-48a7-8795-d6fc19905be4 kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: 'Created pod: model-chainer-5894656ff9-2kqdb' metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:15Z" name: model-chainer-5894656ff9.18a90e6e76080e6f namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 6e609a29-8f0a-498d-8f55-c4ef2b55a354 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: dcd0285e-edf7-43e3-b35b-9e8822b5ba6f kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: Scaled up replica set model-chainer-5894656ff9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:15Z" name: model-chainer.18a90e6e74f77f70 namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 5ecba6bb-5b7a-4f4e-8a5c-dcf64b15f9ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 372acc7e-3810-4fdf-b52c-7772fbb9f784 kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:15Z" name: model-chainer.18a90e6e78d03a7e namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 02652261-bb73-4577-9b66-91b2c7a6ee2c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:04:25.873476Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0065f-6c7bbccfdc-477lq to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:04:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:04:25Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90e7ed4d0b5be namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 3af86908-225d-4b4d-a32f-81114b859d17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-0065f-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90e7edf29ae56 namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 9fdb97a6-9db3-4d6c-ab8c-a1e8209063f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90e7f1150a0e2 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 14ee1e95-0b87-43bd-aa08-24f4337c4849 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0065f} kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90e7f12939535 namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: 61117533-3d13-4019-ba89-166419aa0a51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0065f} kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: 'Created container: sequence-graph-0065f' metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90e7f15cfc43d namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: c814b477-a92e-450f-acde-90892d48f40c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0065f} kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: Started container sequence-graph-0065f metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90e7f164151a4 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 8a7e8a65-0a6f-4ffe-9458-ac916fdd6a5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0065f} kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Stopping container sequence-graph-0065f metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:40Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90ef2045d4048 namespace: kserve-ci-e2e-test resourceVersion: "18751" uid: 4c8dbcdd-44a2-4a90-b1ef-df797cb04f56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0065f} kind: Pod name: sequence-graph-0065f-6c7bbccfdc-477lq namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 93c16c6f-faad-4314-abe0-0789a35a76bb kind: Event lastTimestamp: "2026-04-23T18:13:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:08Z" name: sequence-graph-0065f-6c7bbccfdc-477lq.18a90ef2aa1b9f41 namespace: kserve-ci-e2e-test resourceVersion: "19150" uid: d1067f2e-218d-47d4-8913-d36b7d2ec029 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0065f-6c7bbccfdc namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 6d08a0af-5af0-4f78-940b-1258cd4fa42a kind: Event lastTimestamp: "2026-04-23T18:04:25Z" message: 'Created pod: sequence-graph-0065f-6c7bbccfdc-477lq' metadata: creationTimestamp: "2026-04-23T18:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:25Z" name: sequence-graph-0065f-6c7bbccfdc.18a90e7ed473a5e4 namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 59be4bc8-7853-492e-8d7b-548e544ab4d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0065f namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: a541017f-e535-49e8-96ac-21e1a71e6b34 kind: Event lastTimestamp: "2026-04-23T18:04:25Z" message: Scaled up replica set sequence-graph-0065f-6c7bbccfdc from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:25Z" name: sequence-graph-0065f.18a90e7ed32b8261 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 49d4ffee-17b3-4351-ba67-f0a00af581cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0065f namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: a4ca0f94-98d8-401c-bca8-39a843067976 kind: Event lastTimestamp: "2026-04-23T18:04:25Z" message: InferenceGraph [sequence-graph-0065f] is Ready metadata: creationTimestamp: "2026-04-23T18:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:25Z" name: sequence-graph-0065f.18a90e7edb3c3e5e namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 293d2d9e-e835-4f20-8eb5-58a0413d470b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0065f namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: a4ca0f94-98d8-401c-bca8-39a843067976 kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: 'Failed to update status for InferenceGraph "sequence-graph-0065f": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-0065f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f.18a90e7edef7b9a2 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 810dca20-a179-431a-9b85-68ab6873c5de reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0065f namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: a4ca0f94-98d8-401c-bca8-39a843067976 kind: Event lastTimestamp: "2026-04-23T18:04:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-0065f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:26Z" name: sequence-graph-0065f.18a90e7edef806e6 namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: daa5c4b2-554c-4166-9485-a50648d1d04f reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0065f namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 1cea126f-197a-422c-9051-e8b23c393794 kind: Event lastTimestamp: "2026-04-23T18:05:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:10Z" name: sequence-graph-0065f.18a90e82520ef1d5 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: e5347055-4872-4057-b70e-b9f4e17619df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0065f namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 1cea126f-197a-422c-9051-e8b23c393794 kind: Event lastTimestamp: "2026-04-23T18:05:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:10Z" name: sequence-graph-0065f.18a90e82527edf2d namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 203a7f56-fab9-4ec7-afff-8d9094340110 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:50.774675Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d864a-6c6d7bb99-5j4hm to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:50Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f025b8cef74 namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: 6614f344-156b-4805-9e32-15c0c723abae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:13:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-d864a-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:51Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f026aede682 namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: aaf171ec-adab-4ab2-b484-8b87edd357a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:13:51Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:51Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f02980a58f9 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: 48dd8f9d-4cb8-4bab-91fb-16387f592681 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d864a} kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:13:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:51Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f0299c25f54 namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 3216a42b-c668-48b3-9943-4749f20d8311 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d864a} kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:13:51Z" message: 'Created container: sequence-graph-d864a' metadata: creationTimestamp: "2026-04-23T18:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:51Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f029dd89a8b namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: 040bc225-69ce-46f1-bbde-71a4e62758c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d864a} kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:13:51Z" message: Started container sequence-graph-d864a metadata: creationTimestamp: "2026-04-23T18:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:51Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f029e4544ed namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 7582257f-5432-42af-b8de-7e4891b89689 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d864a} kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: Stopping container sequence-graph-d864a metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:00Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f04b2d3abe4 namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 018e0716-0096-418d-a6e6-f2b74bb30501 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:14:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d864a} kind: Pod name: sequence-graph-d864a-6c6d7bb99-5j4hm namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 379de24a-69bc-42c7-8e6e-bcef40a7af97 kind: Event lastTimestamp: "2026-04-23T18:14:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:14:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:28Z" name: sequence-graph-d864a-6c6d7bb99-5j4hm.18a90f0568054df3 namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: e6c4df7a-7e43-4231-940b-5a4ebf70ccac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d864a-6c6d7bb99 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 25a752e3-706b-49a5-95c0-f27862575130 kind: Event lastTimestamp: "2026-04-23T18:13:50Z" message: 'Created pod: sequence-graph-d864a-6c6d7bb99-5j4hm' metadata: creationTimestamp: "2026-04-23T18:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:50Z" name: sequence-graph-d864a-6c6d7bb99.18a90f025b13c829 namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 0d88adc2-0b4b-4c1b-babe-435cd868c66b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d864a namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: 2c8ac747-d996-4514-8ab1-6c98f0afe340 kind: Event lastTimestamp: "2026-04-23T18:13:50Z" message: Scaled up replica set sequence-graph-d864a-6c6d7bb99 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:50Z" name: sequence-graph-d864a.18a90f025a1ff20d namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 7695da77-a1a0-4397-b8f7-cf263e89faf1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d864a namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: 617fb32a-415a-4fdc-8897-c5b8896a8282 kind: Event lastTimestamp: "2026-04-23T18:13:50Z" message: InferenceGraph [sequence-graph-d864a] is Ready metadata: creationTimestamp: "2026-04-23T18:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:50Z" name: sequence-graph-d864a.18a90f02603a6d80 namespace: kserve-ci-e2e-test resourceVersion: "19684" uid: 5f9f149e-780c-4077-a5e2-130f7c94f165 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:15:01.044747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e1fac-84bd497c8-5xwfd to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:15:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:15:01Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f12b7f91980 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 9f5f72a9-074c-403f-8912-80e6168c36e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:15:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-e1fac-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:01Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f12c54834ab namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: 0239d6e7-e94e-4bbb-99c5-d5c25b0347b4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:15:02Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:15:02Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f12f4813bfc namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: b6c1cac6-d3b4-4657-8c7a-365768b3816d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e1fac} kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:15:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:02Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f12f5be242c namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 9d626c7a-c7c3-420c-a387-1867d83d41be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e1fac} kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:15:02Z" message: 'Created container: sequence-graph-e1fac' metadata: creationTimestamp: "2026-04-23T18:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:02Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f12f85a9400 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 19baf2ae-1a99-495a-90c3-72cc2e8635bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e1fac} kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:15:02Z" message: Started container sequence-graph-e1fac metadata: creationTimestamp: "2026-04-23T18:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:02Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f12f8cbc4d2 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 9d75cbd7-5b97-4dcf-acec-de9b9694c31c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e1fac} kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Stopping container sequence-graph-e1fac metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:15Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f85e1b60c32 namespace: kserve-ci-e2e-test resourceVersion: "24085" uid: 179976e1-2df6-4f9c-9414-5d6e96cb38d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:23:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e1fac} kind: Pod name: sequence-graph-e1fac-84bd497c8-5xwfd namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 4df6da3d-538a-4c59-8197-289f88b09488 kind: Event lastTimestamp: "2026-04-23T18:23:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:44Z" name: sequence-graph-e1fac-84bd497c8-5xwfd.18a90f86abc7496b namespace: kserve-ci-e2e-test resourceVersion: "24623" uid: 0d8e5500-f6b4-4beb-a60d-7a6f385c72aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e1fac-84bd497c8 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: d51532aa-01d5-44a9-9ccf-01144b1b3484 kind: Event lastTimestamp: "2026-04-23T18:15:01Z" message: 'Created pod: sequence-graph-e1fac-84bd497c8-5xwfd' metadata: creationTimestamp: "2026-04-23T18:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:01Z" name: sequence-graph-e1fac-84bd497c8.18a90f12b79011da namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 923f93b7-6adb-4bd0-a949-e2ba1022c370 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e1fac namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: f6dcb325-fbb7-4908-86ae-7fcf71766fcb kind: Event lastTimestamp: "2026-04-23T18:15:01Z" message: Scaled up replica set sequence-graph-e1fac-84bd497c8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:01Z" name: sequence-graph-e1fac.18a90f12b6c93795 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 9d863590-64ba-4eca-aa7f-fda043c5ae5b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e1fac namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 18186cb0-4fdf-4f17-a7c2-8edef266c74b kind: Event lastTimestamp: "2026-04-23T18:15:01Z" message: InferenceGraph [sequence-graph-e1fac] is Ready metadata: creationTimestamp: "2026-04-23T18:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:15:01Z" name: sequence-graph-e1fac.18a90f12bbf5e892 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: a844fe77-e660-453b-bfbc-565192535bef reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e1fac namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: 843374da-86eb-4c64-a59e-73cecc4c156f kind: Event lastTimestamp: "2026-04-23T18:15:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:31Z" name: sequence-graph-e1fac.18a90f163576a414 namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 40f398c4-8179-41fc-abea-2f2d88c6a6a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e1fac namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: 843374da-86eb-4c64-a59e-73cecc4c156f kind: Event lastTimestamp: "2026-04-23T18:15:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:31Z" name: sequence-graph-e1fac.18a90f1635c4d164 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: f17cd89a-125b-42b7-a70c-25fb6299e193 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:24:36.351954Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7efef-547d76f799-6q2j2 to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:24:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:24:36Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a90f98aaf47b53 namespace: kserve-ci-e2e-test resourceVersion: "25140" uid: 12b31fa8-816a-4457-9d6b-e1101a998370 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:36Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:24:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-7efef-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:36Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a90f98b611668f namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: c100b75f-5c69-4cf9-a2f7-bbc429e31c9c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:37Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25143" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:24:37Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:24:37Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a90f98e769d901 namespace: kserve-ci-e2e-test resourceVersion: "25162" uid: 4ff39a50-6d8e-4be5-9bcd-8576242d72f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7efef} kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:24:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:37Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a90f98e8a393ab namespace: kserve-ci-e2e-test resourceVersion: "25164" uid: 562c55c8-c948-4c18-9d38-3f5b0ea78acd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7efef} kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:24:37Z" message: 'Created container: splitter-graph-7efef' metadata: creationTimestamp: "2026-04-23T18:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:37Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a90f98eb648cc8 namespace: kserve-ci-e2e-test resourceVersion: "25166" uid: bacee055-f503-4dfe-9ba3-54fe7c8f9c70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7efef} kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:24:37Z" message: Started container splitter-graph-7efef metadata: creationTimestamp: "2026-04-23T18:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:37Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a90f98ebeaff4f namespace: kserve-ci-e2e-test resourceVersion: "25167" uid: 7f6ecc57-5aa9-46b3-8fa4-2ded5fce8086 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7efef} kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:32:51Z" message: Stopping container splitter-graph-7efef metadata: creationTimestamp: "2026-04-23T18:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:51Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a9100bddd9e332 namespace: kserve-ci-e2e-test resourceVersion: "28892" uid: 9707d04e-2d93-482a-8239-51f2dfc1253c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7efef} kind: Pod name: splitter-graph-7efef-547d76f799-6q2j2 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b9a7dfa6-511e-4872-ad2d-92f0ab085855 kind: Event lastTimestamp: "2026-04-23T18:33:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:18Z" name: splitter-graph-7efef-547d76f799-6q2j2.18a9100c70dfb09b namespace: kserve-ci-e2e-test resourceVersion: "29121" uid: 55b9cdd4-b38a-4ee9-8b41-c5e0f8626100 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7efef-547d76f799 namespace: kserve-ci-e2e-test resourceVersion: "25128" uid: 16e1d8df-d533-403d-aabc-d7d69506c904 kind: Event lastTimestamp: "2026-04-23T18:24:36Z" message: 'Created pod: splitter-graph-7efef-547d76f799-6q2j2' metadata: creationTimestamp: "2026-04-23T18:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:36Z" name: splitter-graph-7efef-547d76f799.18a90f98aa604c2f namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 530c1254-ae38-4789-9669-222301c053ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7efef namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: 05415953-1bdb-4d04-a6b5-d53681d9d0c1 kind: Event lastTimestamp: "2026-04-23T18:24:36Z" message: Scaled up replica set splitter-graph-7efef-547d76f799 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:36Z" name: splitter-graph-7efef.18a90f98a9b9c24f namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: 497f1efc-51c2-4afb-a129-7d54eec221a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7efef namespace: kserve-ci-e2e-test resourceVersion: "25145" uid: 3e3fc91c-4136-4b8e-9952-f846260789d9 kind: Event lastTimestamp: "2026-04-23T18:24:36Z" message: InferenceGraph [splitter-graph-7efef] is Ready metadata: creationTimestamp: "2026-04-23T18:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:36Z" name: splitter-graph-7efef.18a90f98ad9e6e93 namespace: kserve-ci-e2e-test resourceVersion: "25147" uid: b6556d37-f4c7-4aa2-b301-203f55518c23 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7efef namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 3f57c02c-a1db-4b77-b864-e459787a2e76 kind: Event lastTimestamp: "2026-04-23T18:25:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:21Z" name: splitter-graph-7efef.18a90f9c283c4e4b namespace: kserve-ci-e2e-test resourceVersion: "25491" uid: 993af4b0-7576-4d68-95be-d65b60622d5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7efef namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 3f57c02c-a1db-4b77-b864-e459787a2e76 kind: Event lastTimestamp: "2026-04-23T18:25:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:21Z" name: splitter-graph-7efef.18a90f9c28a752ee namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: f03bc334-5818-4c34-bde7-41d72d24ca90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:26.060821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a93f3-c7fd4dc8d-r2jt5 to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f884d46e0ac namespace: kserve-ci-e2e-test resourceVersion: "24312" uid: f70d5c58-b828-4a92-819f-4ea1f7cf7f7b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24315" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f88663a6553 namespace: kserve-ci-e2e-test resourceVersion: "24331" uid: cdc03ee9-cc6e-41ab-b1b4-ef0858d77909 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a93f3} kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f8867833f0f namespace: kserve-ci-e2e-test resourceVersion: "24333" uid: 93ead013-14cf-41c6-9cae-1206b283b590 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a93f3} kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'Created container: splitter-graph-a93f3' metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f886a31df42 namespace: kserve-ci-e2e-test resourceVersion: "24335" uid: afa4621e-a52f-4c09-9947-3c2d36138e9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a93f3} kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: Started container splitter-graph-a93f3 metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f886aa5264c namespace: kserve-ci-e2e-test resourceVersion: "24336" uid: be7c2c6d-43ad-477c-8de8-94afbe7acdb0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a93f3} kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Stopping container splitter-graph-a93f3 metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f8aa674e1a4 namespace: kserve-ci-e2e-test resourceVersion: "24435" uid: fffbc6ce-575d-4ed6-83ba-26afb3c321bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:23:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a93f3} kind: Pod name: splitter-graph-a93f3-c7fd4dc8d-r2jt5 namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 0fca9923-b6f0-43a8-8db9-469992c798e0 kind: Event lastTimestamp: "2026-04-23T18:24:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:03Z" name: splitter-graph-a93f3-c7fd4dc8d-r2jt5.18a90f8b2c9bfbee namespace: kserve-ci-e2e-test resourceVersion: "24786" uid: aa8b6b49-d07d-4050-a1d5-cddc7c000cda reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a93f3-c7fd4dc8d namespace: kserve-ci-e2e-test resourceVersion: "24302" uid: 72255c2c-a9df-4f55-acde-75ca39e8e05e kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'Created pod: splitter-graph-a93f3-c7fd4dc8d-r2jt5' metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3-c7fd4dc8d.18a90f884cdde8a2 namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: 7dc9efb3-8d18-4cc6-9a8e-74a4ea98e012 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "24300" uid: 513d3afc-f84f-45f1-9815-2727aabce2df kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: Scaled up replica set splitter-graph-a93f3-c7fd4dc8d from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3.18a90f884c407587 namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: 8d06f01f-de13-45fc-bf50-ea55d6f39961 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "24321" uid: 168b851d-198c-4547-8901-852c96179bc8 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: InferenceGraph [splitter-graph-a93f3] is Ready metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3.18a90f88509ad2d3 namespace: kserve-ci-e2e-test resourceVersion: "24323" uid: d4f90188-3e84-4e27-ae0b-d394df8871d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "24293" uid: 168b851d-198c-4547-8901-852c96179bc8 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'Failed to update status for InferenceGraph "splitter-graph-a93f3": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-a93f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3.18a90f88533330bd namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: 94629a06-5b25-418e-9be5-9eed2ce0a458 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "24293" uid: 168b851d-198c-4547-8901-852c96179bc8 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-a93f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:26Z" name: splitter-graph-a93f3.18a90f88533378ad namespace: kserve-ci-e2e-test resourceVersion: "24327" uid: 5029680e-3f5b-4c6f-9f2e-6872c3e888e9 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:03:25.848952Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0065f-predictor-76669b4f4-cd5ls to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:03:25Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e70db132793 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 62371bfa-1a2e-42a9-88a1-5c6247de1c3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0065f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:26Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e70e66c277f namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 05e88264-f808-41eb-9593-4677e76cb89f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:03:26Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e7118527d25 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 64dbe3d4-5980-4600-be32-c3269dc82ea0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:26Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e7119911deb namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 47335a46-2c10-496c-b1de-338b0a84a737 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:26Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e711cffae10 namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 92cb7059-98e9-4c3b-9168-497f4f7b93d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:26Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e711d857cf3 namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: b0074ef9-91fe-4573-8fd9-c51851759201 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:26Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e711d919b92 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: a177e2bc-9b31-4448-ade0-b7499d1c5290 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e7122a3a6f3 namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 1d169778-14db-4aee-a6ff-a6e89cdf9341 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:03:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:27Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e7123175810 namespace: kserve-ci-e2e-test resourceVersion: "14525" uid: 8c5af522-47b7-49dc-8cf9-e5d50fecb4b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:04:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:04Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90e718d889ee1 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 609ed1cd-71b1-4f21-babe-a98f1c667d1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90ef20a07149d namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: eb80c61b-ed14-40f9-85b0-98127a485600 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 59ee2c69-d1b8-4c6f-a69b-20e2cf2de714 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-0065f-predictor-76669b4f4-cd5ls.18a90ef20a07b708 namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: 48a54b28-db3a-41e8-b4bb-a3d3c9898fd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0065f-predictor-76669b4f4 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 2c1f674c-1fce-4057-8b83-b8f1c10c9405 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: 'Created pod: success-200-isvc-0065f-predictor-76669b4f4-cd5ls' metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:25Z" name: success-200-isvc-0065f-predictor-76669b4f4.18a90e70daa8f8a5 namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: c47f8bcc-bc74-4cb2-9c67-950e7301b2d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 85bdcd11-7a7a-4f20-82d7-354936adf252 kind: Event lastTimestamp: "2026-04-23T18:03:25Z" message: Scaled up replica set success-200-isvc-0065f-predictor-76669b4f4 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:25Z" name: success-200-isvc-0065f-predictor.18a90e70d8639d7e namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 091d58d1-8ddf-4e38-a864-a29981750744 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: e78d159c-ed02-4627-b8b8-d9369cb6b38f kind: Event lastTimestamp: "2026-04-23T18:03:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:55Z" name: success-200-isvc-0065f-predictor.18a90e7457757194 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 3e1e53c8-9c85-409c-bb57-78f173a89621 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: e78d159c-ed02-4627-b8b8-d9369cb6b38f kind: Event lastTimestamp: "2026-04-23T18:03:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:55Z" name: success-200-isvc-0065f-predictor.18a90e7457e808f6 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: fa2a547b-4ca4-498f-b930-eca572af0523 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:04:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14678" uid: e78d159c-ed02-4627-b8b8-d9369cb6b38f kind: Event lastTimestamp: "2026-04-23T18:04:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:25Z" name: success-200-isvc-0065f-predictor.18a90e7b55e13d4d namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 172b88a0-3254-42d3-8389-acb3f3303ba7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:04:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0065f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14678" uid: e78d159c-ed02-4627-b8b8-d9369cb6b38f kind: Event lastTimestamp: "2026-04-23T18:04:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:25Z" name: success-200-isvc-0065f-predictor.18a90e7b564f84f5 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: de87088b-34f8-4bee-9838-efe28132fc34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:03:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0065f namespace: kserve-ci-e2e-test resourceVersion: "14403" uid: f0144670-1a3d-470a-b6e4-c8f8c07631f1 kind: Event lastTimestamp: "2026-04-23T18:04:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:14Z" name: success-200-isvc-0065f.18a90e70d85a25b0 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 69d603c6-8186-4311-b574-bf438cb8c1ce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0065f namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: f0144670-1a3d-470a-b6e4-c8f8c07631f1 kind: Event lastTimestamp: "2026-04-23T18:04:14Z" message: InferenceService [success-200-isvc-0065f] is Ready metadata: creationTimestamp: "2026-04-23T18:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:04:14Z" name: success-200-isvc-0065f.18a90e7c4944f428 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 52d9d508-9869-4a17-b119-7b12416537a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:00.738061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18349" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8bcdb6d5a namespace: kserve-ci-e2e-test resourceVersion: "18359" uid: 98f751aa-f713-4b25-9974-ea3ae342292b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18368" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8d6230db8 namespace: kserve-ci-e2e-test resourceVersion: "18399" uid: 21b87fb0-d8d1-4dcf-a6b7-0e7cb993dcc6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8d772968a namespace: kserve-ci-e2e-test resourceVersion: "18401" uid: 90bcfa33-782a-4264-9c01-34b4fa7675ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8da311bff namespace: kserve-ci-e2e-test resourceVersion: "18402" uid: b3a549bf-d8c8-48f8-8b01-68d91059787c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8daaab21c namespace: kserve-ci-e2e-test resourceVersion: "18403" uid: 23bf18ed-9913-4a49-a631-9464fc282dd1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8db118c74 namespace: kserve-ci-e2e-test resourceVersion: "18405" uid: c6a436d6-3fcc-48c4-b915-4d1c2023ec9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8df3653e4 namespace: kserve-ci-e2e-test resourceVersion: "18407" uid: 4b5d0e76-eb1b-48b8-8637-dd0bf8836e92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:01Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee8dfb5a319 namespace: kserve-ci-e2e-test resourceVersion: "18408" uid: 14741eca-78cb-48df-bcef-516cdea34736 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:12:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:38Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ee9245f8054 namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: 8db657ff-ebe3-41be-9302-7ada5d4dc95b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ef9174eb48c namespace: kserve-ci-e2e-test resourceVersion: "19227" uid: ec48d327-3610-4b39-bbbd-987f2de4f282 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ef9174f71d8 namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: eb50de41-f604-4fc5-bf8a-047f4e568239 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 498d0cae-4e8f-4ad9-9039-ff9d5f34e74a kind: Event lastTimestamp: "2026-04-23T18:13:13Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:13Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8.18a90ef9ac7986b5 namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: fd98d6d3-a534-4325-9031-c308b2c12b1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-226e9-predictor-5c7d88cfd8 namespace: kserve-ci-e2e-test resourceVersion: "18328" uid: b02cfeb7-58d1-47f2-9bd3-e734cf569a79 kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'Created pod: success-200-isvc-226e9-predictor-5c7d88cfd8-hpjr8' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-226e9-predictor-5c7d88cfd8.18a90ee8bc50f8a8 namespace: kserve-ci-e2e-test resourceVersion: "18352" uid: be0cc302-c34c-40d5-8415-165ecf1731e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 4898c567-dafd-4606-b056-6e392839e1db kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Scaled up replica set success-200-isvc-226e9-predictor-5c7d88cfd8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-226e9-predictor.18a90ee8b8cfe3e9 namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: 1f903053-31fe-454b-82c7-15e623978011 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18327" uid: d88dffb5-0043-4f86-b36d-0e9cccd50b37 kind: Event lastTimestamp: "2026-04-23T18:12:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:30Z" name: success-200-isvc-226e9-predictor.18a90eec375eaef9 namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 970fbff8-6197-435a-b8a6-c45366d383fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18327" uid: d88dffb5-0043-4f86-b36d-0e9cccd50b37 kind: Event lastTimestamp: "2026-04-23T18:12:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:30Z" name: success-200-isvc-226e9-predictor.18a90eec37c382da namespace: kserve-ci-e2e-test resourceVersion: "18658" uid: 4cdc690a-6944-47ce-86ac-d0294ad2e21e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18544" uid: d88dffb5-0043-4f86-b36d-0e9cccd50b37 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:00Z" name: success-200-isvc-226e9-predictor.18a90ef335a04fc3 namespace: kserve-ci-e2e-test resourceVersion: "19059" uid: d9a9d558-3524-425a-92a8-8fec8e57f48d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-226e9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18544" uid: d88dffb5-0043-4f86-b36d-0e9cccd50b37 kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:00Z" name: success-200-isvc-226e9-predictor.18a90ef336192ed8 namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: d17357b9-67e4-4c71-9154-189433e0a2a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: b3372543-fced-4ced-85a9-c09d6ce73e0b kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:48Z" name: success-200-isvc-226e9.18a90ee8b8abe4f2 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: faad7633-54ca-47a4-af06-2be95856398a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: b3372543-fced-4ced-85a9-c09d6ce73e0b kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-226e9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-226e9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-226e9.18a90ee8b8fe3bb4 namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 3161740f-4ca4-4c04-940f-cfc189992db4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: b3372543-fced-4ced-85a9-c09d6ce73e0b kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-226e9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-226e9.18a90ee8b8fe8b28 namespace: kserve-ci-e2e-test resourceVersion: "18334" uid: eb3180ca-66f1-4b81-8677-1a1ce7aa9be9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-226e9 namespace: kserve-ci-e2e-test resourceVersion: "18945" uid: b3372543-fced-4ced-85a9-c09d6ce73e0b kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: InferenceService [success-200-isvc-226e9] is Ready metadata: creationTimestamp: "2026-04-23T18:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:48Z" name: success-200-isvc-226e9.18a90ef3e091c1ab namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: 3d254a10-ac78-4100-80fa-744e39dba294 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:02:45.911144Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13915" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3d086-predictor-67ccc5b954-l2q2w to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e678e989275 namespace: kserve-ci-e2e-test resourceVersion: "13922" uid: aa2a5c99-0527-460b-93c8-e3277e2ffefa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67a7fa7cb1 namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: 593a7046-9718-4653-9552-5975d853c537 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67a950cd77 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: ffc91221-5098-4279-aede-29054e434c3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67abfe4af2 namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: 36086cfa-9783-4866-80b5-f942f2ee3737 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67ac6f7aea namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 26e7b6ee-cfc3-4776-97d0-a32034023758 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67ac7c4791 namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: d5bf3bbc-087d-496f-8af4-835e2d397677 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67af8f02dc namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 6c6a47b7-39ca-4000-8f3e-44a1d0a5be8c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:02:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:46Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67affdd543 namespace: kserve-ci-e2e-test resourceVersion: "13974" uid: b264ad76-5f11-4be8-be70-4c226bbf8e89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:12:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:03Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90e67f968151d namespace: kserve-ci-e2e-test resourceVersion: "18450" uid: d1eee81d-3bf5-4faa-853b-cbccb4abcd95 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90ee8b84ff004 namespace: kserve-ci-e2e-test resourceVersion: "18326" uid: ae472cca-2180-445d-b957-3a8570126a5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:00Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90ee8b8508f2c namespace: kserve-ci-e2e-test resourceVersion: "18329" uid: 358e7b08-c1d5-4931-94a2-d92baf8d579f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: 0a9cc4a6-864f-450c-a9b2-3a626d274d3b kind: Event lastTimestamp: "2026-04-23T18:12:03Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:03Z" name: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w.18a90ee96db8f6b0 namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: b5c04237-af3e-4a73-b104-12a209efc782 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3d086-predictor-67ccc5b954 namespace: kserve-ci-e2e-test resourceVersion: "13900" uid: 9e188060-7f77-48d4-b6de-caab0329121c kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'Created pod: success-200-isvc-3d086-predictor-67ccc5b954-l2q2w' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-3d086-predictor-67ccc5b954.18a90e678e363cc4 namespace: kserve-ci-e2e-test resourceVersion: "13920" uid: 7ed883d8-1ebd-4ee7-85b0-83eeae56381e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: 3518235a-bba8-411a-90b3-4e7ccecc388c kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Scaled up replica set success-200-isvc-3d086-predictor-67ccc5b954 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-3d086-predictor.18a90e678c0d2143 namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: 7bab9a92-0dc3-41ba-8eae-e05eed63eb7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: f53569e5-07e2-41f3-bf73-636d098a970d kind: Event lastTimestamp: "2026-04-23T18:03:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:00Z" name: success-200-isvc-3d086-predictor.18a90e6b0aae429f namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 7aa990a7-5029-4d63-b328-3479936e0282 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: f53569e5-07e2-41f3-bf73-636d098a970d kind: Event lastTimestamp: "2026-04-23T18:03:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:00Z" name: success-200-isvc-3d086-predictor.18a90e6b0b1a3e74 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 7acad9db-c1b6-4d5c-873a-0ea8b4f121e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:03:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: f53569e5-07e2-41f3-bf73-636d098a970d kind: Event lastTimestamp: "2026-04-23T18:04:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:00Z" name: success-200-isvc-3d086-predictor.18a90e6e8a1937d2 namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: e0853d0e-5843-48ca-86fd-9a8d4bbcf0fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:03:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d086-predictor namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: f53569e5-07e2-41f3-bf73-636d098a970d kind: Event lastTimestamp: "2026-04-23T18:04:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:00Z" name: success-200-isvc-3d086-predictor.18a90e6e8aad48f6 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: ad207bf8-f244-4ced-8e0e-7eaced62b309 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: 147a185b-f110-42bc-83b4-62a878a5e8ac kind: Event lastTimestamp: "2026-04-23T18:03:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:33Z" name: success-200-isvc-3d086.18a90e678c1ecb66 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 9eb39806-6beb-4bbb-bba8-44a0a0bfde7a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: 147a185b-f110-42bc-83b4-62a878a5e8ac kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-3d086": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3d086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-3d086.18a90e678cad0000 namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 5cfd2f2a-00e7-4f4a-b843-07ad64eb5023 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: 147a185b-f110-42bc-83b4-62a878a5e8ac kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3d086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-3d086.18a90e678cad8f48 namespace: kserve-ci-e2e-test resourceVersion: "13909" uid: 3d9be971-4a45-4ff3-9c5b-b4c0ff856e37 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 147a185b-f110-42bc-83b4-62a878a5e8ac kind: Event lastTimestamp: "2026-04-23T18:03:33Z" message: InferenceService [success-200-isvc-3d086] is Ready metadata: creationTimestamp: "2026-04-23T18:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:33Z" name: success-200-isvc-3d086.18a90e72b4ce11b9 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 2fd2eab7-3a25-455a-9cb8-51ed036f02a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:10.998272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5c6fb-predictor-85d68895bd-htcgs to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef918b1207b namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: 35947dc7-3b31-4eee-a81b-232c12d45db9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef932a54811 namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 54e24a84-872c-49bf-ae7a-61f13b0b5b12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef93403a1bb namespace: kserve-ci-e2e-test resourceVersion: "19295" uid: bab8d5d0-fdf6-404c-94a9-747fac31f554 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef936f39e48 namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: c185a263-4ddd-4e28-8fdb-538317f51dde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef937690944 namespace: kserve-ci-e2e-test resourceVersion: "19298" uid: d8c08935-6b4e-40a4-a491-f603e0defd4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef93774c6a1 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 63272479-7bec-4d82-acb0-82c190f5feb9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef93a540d3c namespace: kserve-ci-e2e-test resourceVersion: "19303" uid: a5941255-177a-455f-8696-cb87c01e613d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef93ac66be3 namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 0e1d14d8-51ae-43f9-81c8-8411d082edb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90ef942f7714f namespace: kserve-ci-e2e-test resourceVersion: "23684" uid: b33e24e3-3471-4471-9d1f-a03430c829b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:22:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:25Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90f7a4f7259bf namespace: kserve-ci-e2e-test resourceVersion: "23580" uid: 4cd85760-b5a4-4f6a-bf77-0f5e4d452fc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:22:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:25Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90f7a4f735985 namespace: kserve-ci-e2e-test resourceVersion: "23582" uid: fbe108ef-3ebc-448e-b3f6-d5bbf1c91d25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: aeb4642d-f56a-442e-9c6e-e2354973f7cb kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs.18a90f7ab759cb60 namespace: kserve-ci-e2e-test resourceVersion: "23683" uid: 04721a41-839a-4730-a806-cbde9afc09c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5c6fb-predictor-85d68895bd namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 211692d2-6376-448a-b1e5-4342c7154a26 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'Created pod: success-200-isvc-5c6fb-predictor-85d68895bd-htcgs' metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-5c6fb-predictor-85d68895bd.18a90ef918335913 namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: 8cefa4b5-08ee-40ea-9d0d-786e8229d3a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: e1949ace-1aa7-42cf-95eb-5b6275b84a01 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Scaled up replica set success-200-isvc-5c6fb-predictor-85d68895bd from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-5c6fb-predictor.18a90ef91665859c namespace: kserve-ci-e2e-test resourceVersion: "19221" uid: d8d23c22-8691-42ef-954f-63f859068369 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: d98b26de-5068-4293-92b3-3edcd6a803c5 kind: Event lastTimestamp: "2026-04-23T18:13:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:40Z" name: success-200-isvc-5c6fb-predictor.18a90efc9543bd50 namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: d6af5f5e-4d05-4d46-a871-f0683d01e1c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: d98b26de-5068-4293-92b3-3edcd6a803c5 kind: Event lastTimestamp: "2026-04-23T18:13:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:41Z" name: success-200-isvc-5c6fb-predictor.18a90efc95bcd756 namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: ee6365c7-e428-4032-b672-ad0e76464442 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: d98b26de-5068-4293-92b3-3edcd6a803c5 kind: Event lastTimestamp: "2026-04-23T18:14:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:26Z" name: success-200-isvc-5c6fb-predictor.18a90f03938c01c5 namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 0b69f879-d4f5-4b32-a961-00cb7b73bcda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c6fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: d98b26de-5068-4293-92b3-3edcd6a803c5 kind: Event lastTimestamp: "2026-04-23T18:14:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:26Z" name: success-200-isvc-5c6fb-predictor.18a90f0393ff8e5b namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: fbfdb8cd-84de-4977-9bd4-54aaf53ef02d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: d675bfc5-9cfa-4ff5-84ee-b304673c592f kind: Event lastTimestamp: "2026-04-23T18:13:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:57Z" name: success-200-isvc-5c6fb.18a90ef9166d5f29 namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: 9f1ae26f-177f-41f9-a2bf-9faa26feaf1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: d675bfc5-9cfa-4ff5-84ee-b304673c592f kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-5c6fb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5c6fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-5c6fb.18a90ef91717eefb namespace: kserve-ci-e2e-test resourceVersion: "19225" uid: e2c89bbc-7092-491c-96d8-45975570e258 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: d675bfc5-9cfa-4ff5-84ee-b304673c592f kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5c6fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-5c6fb.18a90ef9171844ac namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 7d23f480-4cda-4057-800a-54ee2e2084cd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c6fb namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: d675bfc5-9cfa-4ff5-84ee-b304673c592f kind: Event lastTimestamp: "2026-04-23T18:13:57Z" message: InferenceService [success-200-isvc-5c6fb] is Ready metadata: creationTimestamp: "2026-04-23T18:13:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:57Z" name: success-200-isvc-5c6fb.18a90f03ff3ec0d1 namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: 335da213-2d5e-4657-91ab-864b6e97199b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:36.294871Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24484" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7efef-predictor-7d574cb67d-sqqwx to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8aaf4611d7 namespace: kserve-ci-e2e-test resourceVersion: "24489" uid: 29c94f5d-37c1-4b90-acdd-c414d5493848 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7efef-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8abb01210a namespace: kserve-ci-e2e-test resourceVersion: "24523" uid: 777f1048-15c0-4631-a1d4-f73eac57bbbf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24493" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8aec2bb8b0 namespace: kserve-ci-e2e-test resourceVersion: "24538" uid: 028d7be5-a8e3-49e7-94fb-3768c9d9d7f3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8aed850f20 namespace: kserve-ci-e2e-test resourceVersion: "24540" uid: 35d721c8-c086-48ae-beba-e1313ae1f9ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8af0589aa9 namespace: kserve-ci-e2e-test resourceVersion: "24541" uid: 77afaff1-5d6f-4933-b42c-2bda4953dfc1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8af0ccb049 namespace: kserve-ci-e2e-test resourceVersion: "24542" uid: 8d68ad3f-1edb-4cae-b1d6-206a91b7cca7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8af0d9cc98 namespace: kserve-ci-e2e-test resourceVersion: "24543" uid: 78bb1fae-2f83-4afe-9125-ce1b9a71d760 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8af48d302a namespace: kserve-ci-e2e-test resourceVersion: "24547" uid: f34472ff-99b5-45d7-9be3-abd4bab9a86e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:23:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:37Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8af500ac3a namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 13d45017-872f-48ee-827c-f122957c3aa3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:23:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:32:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:54Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a90f8b419af343 namespace: kserve-ci-e2e-test resourceVersion: "28949" uid: f4723f93-0d96-419c-8823-4f5e6764707e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:32:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:51Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a9100bf5283d38 namespace: kserve-ci-e2e-test resourceVersion: "28923" uid: d76cdd2d-1af3-480e-b788-abf421a5156d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:32:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:51Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a9100bf528b57e namespace: kserve-ci-e2e-test resourceVersion: "28924" uid: fbb0957d-59d9-43f2-8b83-c446b75cf5a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 4bfc3557-c73f-4baf-b97e-8f69d925fafc kind: Event lastTimestamp: "2026-04-23T18:32:54Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:54Z" name: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx.18a9100cb5ffe7d4 namespace: kserve-ci-e2e-test resourceVersion: "28948" uid: 9e9a7a5b-a256-4ed3-8eb6-bd1b7901ad4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7efef-predictor-7d574cb67d namespace: kserve-ci-e2e-test resourceVersion: "24467" uid: f378f077-8f34-4a98-b0d5-84b3f94b67df kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'Created pod: success-200-isvc-7efef-predictor-7d574cb67d-sqqwx' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-7efef-predictor-7d574cb67d.18a90f8aaee087fc namespace: kserve-ci-e2e-test resourceVersion: "24486" uid: c00a9ea1-60cd-432e-a576-b8556fa180af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24461" uid: a5d08d69-d700-4c0b-89ea-43065f47cd89 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Scaled up replica set success-200-isvc-7efef-predictor-7d574cb67d from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-7efef-predictor.18a90f8aacb37a1b namespace: kserve-ci-e2e-test resourceVersion: "24470" uid: caab02d4-6369-4b08-a9e8-5eb045f54142 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: 61f43f7d-dc41-4e25-a886-f0cf010a0f97 kind: Event lastTimestamp: "2026-04-23T18:23:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:51Z" name: success-200-isvc-7efef-predictor.18a90f8e2b623d1d namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: c9b99cfd-ed0a-45a4-9251-67b7a97233c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: 61f43f7d-dc41-4e25-a886-f0cf010a0f97 kind: Event lastTimestamp: "2026-04-23T18:23:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:51Z" name: success-200-isvc-7efef-predictor.18a90f8e2bbe65f5 namespace: kserve-ci-e2e-test resourceVersion: "24694" uid: a236aba7-5167-4807-9f51-2584719ad2ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: 61f43f7d-dc41-4e25-a886-f0cf010a0f97 kind: Event lastTimestamp: "2026-04-23T18:24:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:51Z" name: success-200-isvc-7efef-predictor.18a90f91aae5600f namespace: kserve-ci-e2e-test resourceVersion: "25277" uid: 2dfc1824-b921-4589-821f-1fa5f4a7d450 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7efef-predictor namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: 61f43f7d-dc41-4e25-a886-f0cf010a0f97 kind: Event lastTimestamp: "2026-04-23T18:24:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:51Z" name: success-200-isvc-7efef-predictor.18a90f91ab57ac22 namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: 4248fd15-eab8-49b7-a8b7-353a1fb9513c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "24439" uid: f417b4c9-e851-4d6b-aa5c-b26d616a21b9 kind: Event lastTimestamp: "2026-04-23T18:24:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:24Z" name: success-200-isvc-7efef.18a90f8aacbb004e namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: c3259acc-80a4-467d-87b1-73f2776b6a10 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "24439" uid: f417b4c9-e851-4d6b-aa5c-b26d616a21b9 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-7efef": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7efef": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-7efef.18a90f8aad4cabe3 namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e8c0ef44-a011-4db7-a4e3-4d911711ea2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "24439" uid: f417b4c9-e851-4d6b-aa5c-b26d616a21b9 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7efef": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-7efef.18a90f8aad4d0643 namespace: kserve-ci-e2e-test resourceVersion: "24479" uid: 9006698f-284d-42f7-b93e-431d075821db reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7efef namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: f417b4c9-e851-4d6b-aa5c-b26d616a21b9 kind: Event lastTimestamp: "2026-04-23T18:24:24Z" message: InferenceService [success-200-isvc-7efef] is Ready metadata: creationTimestamp: "2026-04-23T18:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:24Z" name: success-200-isvc-7efef.18a90f95fcc3e79a namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: 05c2ed52-890a-4a57-8d57-8810ea02614d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:15.808782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24121" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f85ea352c4e namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: 9afd71a0-38d9-4d70-a0ff-5bedbe2ce66f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-87aee-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f85f1867f73 namespace: kserve-ci-e2e-test resourceVersion: "24167" uid: e03cf5b5-fa96-4b19-afec-c50d4386c3d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24142" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f862719233d namespace: kserve-ci-e2e-test resourceVersion: "24196" uid: f18bb37f-6337-4c01-93fb-11eee3d3991e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f86287f963d namespace: kserve-ci-e2e-test resourceVersion: "24198" uid: 01e47ec6-31bb-4701-a6b3-eca84194a8b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f862b8b84ca namespace: kserve-ci-e2e-test resourceVersion: "24200" uid: f023bb5b-0f67-4bbc-b94e-95dedcf5c49c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f862c003cb2 namespace: kserve-ci-e2e-test resourceVersion: "24201" uid: d903142c-233d-4f7c-9b43-4da03a786725 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f862c0d3aec namespace: kserve-ci-e2e-test resourceVersion: "24202" uid: 48f3be2b-3804-46ab-b22c-785baed71926 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f86302b0a7d namespace: kserve-ci-e2e-test resourceVersion: "24208" uid: a055cf6d-aedf-4bd3-a27c-06bddfe5e8e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:16Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f8630b3e6d4 namespace: kserve-ci-e2e-test resourceVersion: "24209" uid: 07f75047-fb87-434c-b6ed-3b2a94664529 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:23:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:23:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:54Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a90f86956385a0 namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: ab26cd89-b97b-4707-b0d0-dbba616c8648 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:40:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:35Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a91077ea043633 namespace: kserve-ci-e2e-test resourceVersion: "32125" uid: 8f72ea16-c893-4d81-91d3-2e97dbf31437 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 5799e9c5-f75b-4d91-b7b2-41b837c8f47a kind: Event lastTimestamp: "2026-04-23T18:40:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:35Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5.18a91077ea04c4c5 namespace: kserve-ci-e2e-test resourceVersion: "32126" uid: a37d6f80-90c9-4bbf-a1c1-547b79e747e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-87aee-predictor-5f8bdb75dd namespace: kserve-ci-e2e-test resourceVersion: "24109" uid: a1429660-4fd6-4e92-aef6-5119bce9fcae kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'Created pod: success-200-isvc-87aee-predictor-5f8bdb75dd-m72f5' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-87aee-predictor-5f8bdb75dd.18a90f85e9ccb211 namespace: kserve-ci-e2e-test resourceVersion: "24123" uid: 6ccbfbba-115b-4373-9d16-9d9de7ae7ba6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24106" uid: e5978802-ea63-4de5-adaf-53b8096013d6 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Scaled up replica set success-200-isvc-87aee-predictor-5f8bdb75dd from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-87aee-predictor.18a90f85e7a7a75c namespace: kserve-ci-e2e-test resourceVersion: "24112" uid: f94b3f44-199e-4073-8379-0c72a03093e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24108" uid: 325614b8-ec6c-479d-8900-9afe41dc16d2 kind: Event lastTimestamp: "2026-04-23T18:23:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:45Z" name: success-200-isvc-87aee-predictor.18a90f8966432039 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: f8c028ee-a342-4e54-b1f0-73a879dca28b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24108" uid: 325614b8-ec6c-479d-8900-9afe41dc16d2 kind: Event lastTimestamp: "2026-04-23T18:23:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:45Z" name: success-200-isvc-87aee-predictor.18a90f8966b87b70 namespace: kserve-ci-e2e-test resourceVersion: "24643" uid: 1979afaa-49ba-4049-a39c-f27ccf80d518 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24372" uid: 325614b8-ec6c-479d-8900-9afe41dc16d2 kind: Event lastTimestamp: "2026-04-23T18:24:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:30Z" name: success-200-isvc-87aee-predictor.18a90f90645a1091 namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 0941fdea-85b9-4f8f-81f3-7c5a6cb5ca92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87aee-predictor namespace: kserve-ci-e2e-test resourceVersion: "24372" uid: 325614b8-ec6c-479d-8900-9afe41dc16d2 kind: Event lastTimestamp: "2026-04-23T18:24:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:30Z" name: success-200-isvc-87aee-predictor.18a90f9064c74537 namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: d20c968c-9f7d-4257-97bb-7fbc0cc9302c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24088" uid: ab1dfbd6-8922-459d-b630-e17ab577d971 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-87aee.18a90f85e7754ac2 namespace: kserve-ci-e2e-test resourceVersion: "24815" uid: 6a494e76-1a56-445c-9103-d19dff9195e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24088" uid: ab1dfbd6-8922-459d-b630-e17ab577d971 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-87aee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87aee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-87aee.18a90f85e8084c55 namespace: kserve-ci-e2e-test resourceVersion: "24114" uid: 51a4922b-7e85-4422-88cd-b19444c5e572 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24088" uid: ab1dfbd6-8922-459d-b630-e17ab577d971 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87aee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-87aee.18a90f85e80896a4 namespace: kserve-ci-e2e-test resourceVersion: "24116" uid: 14bfa8d4-2047-47db-810d-44dd98a104f2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87aee namespace: kserve-ci-e2e-test resourceVersion: "24813" uid: ab1dfbd6-8922-459d-b630-e17ab577d971 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: InferenceService [success-200-isvc-87aee] is Ready metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-87aee.18a90f9150e1d4d7 namespace: kserve-ci-e2e-test resourceVersion: "24814" uid: 2d3a4f4f-ba8a-4d42-a55d-ec13937ccfa2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:22:26.052918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23602" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a93f3-predictor-5c444dc64-lkwsq to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:22:26Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a5486f09c namespace: kserve-ci-e2e-test resourceVersion: "23614" uid: d2ccbdf2-a791-449b-a158-c98c51a81b1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a93f3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a5e45f1e4 namespace: kserve-ci-e2e-test resourceVersion: "23641" uid: 2fb4c0c7-92e1-47f1-907b-ca3b547a3d1f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23623" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a9142b4e8 namespace: kserve-ci-e2e-test resourceVersion: "23666" uid: 2e4dca8a-c683-40e4-915b-31b6011c604d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a92a8efb0 namespace: kserve-ci-e2e-test resourceVersion: "23668" uid: 3d4c02f4-7428-4722-b6e1-da804394bdf9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a956f0f7c namespace: kserve-ci-e2e-test resourceVersion: "23670" uid: dbebb7e5-5689-4184-b084-6c5e97fe2752 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a95e50ee2 namespace: kserve-ci-e2e-test resourceVersion: "23672" uid: 1f5db087-5f2c-4e7a-99ed-492baebf7d3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a95f116e8 namespace: kserve-ci-e2e-test resourceVersion: "23673" uid: 512e568c-d16b-474b-87db-8542a938a422 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a98ea101b namespace: kserve-ci-e2e-test resourceVersion: "23674" uid: 1a81b48b-e57c-4d63-985d-2a8088b60198 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:22:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:27Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7a995c9a32 namespace: kserve-ci-e2e-test resourceVersion: "23675" uid: 3742571f-d09d-457c-a747-7c69cc73c46c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:23:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:04Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f7ae6ed9ec2 namespace: kserve-ci-e2e-test resourceVersion: "23969" uid: 2e966b25-d73a-4c25-ad52-0b5c1b199295 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f8aae65fe04 namespace: kserve-ci-e2e-test resourceVersion: "24482" uid: 5612e8a4-2f48-4f07-903e-badb800e150f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f8aae664b1e namespace: kserve-ci-e2e-test resourceVersion: "24485" uid: 530df1e4-084d-4856-ab0b-bcf3485e173b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq namespace: kserve-ci-e2e-test resourceVersion: "23606" uid: f12765e8-eacd-45c5-99b4-6c3db88f3198 kind: Event lastTimestamp: "2026-04-23T18:23:40Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-23T18:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:40Z" name: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq.18a90f8baacbb6fb namespace: kserve-ci-e2e-test resourceVersion: "24599" uid: 24791007-eb5b-4d67-9218-1d55617c329c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a93f3-predictor-5c444dc64 namespace: kserve-ci-e2e-test resourceVersion: "23577" uid: bcde463b-8eea-4a4c-af4f-67960ef2664e kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'Created pod: success-200-isvc-a93f3-predictor-5c444dc64-lkwsq' metadata: creationTimestamp: "2026-04-23T18:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:26Z" name: success-200-isvc-a93f3-predictor-5c444dc64.18a90f7a53ec171a namespace: kserve-ci-e2e-test resourceVersion: "23605" uid: e2848c7f-f4d5-4f3d-9bc6-b957e9e2f3e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23573" uid: f4deda33-b74c-4d03-b1a8-318283dcb360 kind: Event lastTimestamp: "2026-04-23T18:22:25Z" message: Scaled up replica set success-200-isvc-a93f3-predictor-5c444dc64 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:25Z" name: success-200-isvc-a93f3-predictor.18a90f7a4f7002e1 namespace: kserve-ci-e2e-test resourceVersion: "23579" uid: 2d177dd4-5d13-4808-96ab-a4ac47b2b1bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23576" uid: 66b65a9d-d953-4efe-860f-b62dc2ae9ee6 kind: Event lastTimestamp: "2026-04-23T18:22:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:56Z" name: success-200-isvc-a93f3-predictor.18a90f7dce8a86aa namespace: kserve-ci-e2e-test resourceVersion: "23906" uid: 3a085572-285e-4389-9783-a9f8803695bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23576" uid: 66b65a9d-d953-4efe-860f-b62dc2ae9ee6 kind: Event lastTimestamp: "2026-04-23T18:22:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:56Z" name: success-200-isvc-a93f3-predictor.18a90f7dceee6710 namespace: kserve-ci-e2e-test resourceVersion: "23907" uid: 33e914a5-2a41-48ad-a20b-bf69ee2b8d83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23791" uid: 66b65a9d-d953-4efe-860f-b62dc2ae9ee6 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:26Z" name: success-200-isvc-a93f3-predictor.18a90f84cd00fe96 namespace: kserve-ci-e2e-test resourceVersion: "24304" uid: 6a5957c1-d815-4d70-b0ee-18b1182b0844 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a93f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "23791" uid: 66b65a9d-d953-4efe-860f-b62dc2ae9ee6 kind: Event lastTimestamp: "2026-04-23T18:23:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:26Z" name: success-200-isvc-a93f3-predictor.18a90f84cd6e8de7 namespace: kserve-ci-e2e-test resourceVersion: "24309" uid: db1c9d9c-3405-4f05-b5eb-d882451e4445 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "23553" uid: 4f6aa412-1c68-45da-83e4-714f34c52b5a kind: Event lastTimestamp: "2026-04-23T18:23:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:14Z" name: success-200-isvc-a93f3.18a90f7a4f592004 namespace: kserve-ci-e2e-test resourceVersion: "24060" uid: 78a09408-65b0-4c80-81ed-3693c8560c46 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "23553" uid: 4f6aa412-1c68-45da-83e4-714f34c52b5a kind: Event lastTimestamp: "2026-04-23T18:22:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-a93f3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a93f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:25Z" name: success-200-isvc-a93f3.18a90f7a50084284 namespace: kserve-ci-e2e-test resourceVersion: "23583" uid: 07e05a32-cdff-4382-8b3b-21b6fb6a71ae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "23553" uid: 4f6aa412-1c68-45da-83e4-714f34c52b5a kind: Event lastTimestamp: "2026-04-23T18:22:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a93f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:25Z" name: success-200-isvc-a93f3.18a90f7a5008bd5b namespace: kserve-ci-e2e-test resourceVersion: "23584" uid: 7bc8c777-41a2-4305-b29e-34c02c9addcf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a93f3 namespace: kserve-ci-e2e-test resourceVersion: "24058" uid: 4f6aa412-1c68-45da-83e4-714f34c52b5a kind: Event lastTimestamp: "2026-04-23T18:23:14Z" message: InferenceService [success-200-isvc-a93f3] is Ready metadata: creationTimestamp: "2026-04-23T18:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:14Z" name: success-200-isvc-a93f3.18a90f85a2306a9f namespace: kserve-ci-e2e-test resourceVersion: "24059" uid: 4a843fc2-7532-4a8b-a609-867c7854fdc5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:40.728674Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef20c7bbff4 namespace: kserve-ci-e2e-test resourceVersion: "18791" uid: 23251440-6423-45ed-b4b8-aa59feb5f5e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d864a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:41Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef214f45edd namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: 9776ca95-c5f6-40dd-b416-21a57713e62e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18802" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef27ef496f4 namespace: kserve-ci-e2e-test resourceVersion: "18867" uid: add0f014-72d0-4edc-995d-e65b191eaf8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef2807025e8 namespace: kserve-ci-e2e-test resourceVersion: "18869" uid: 5ddf3e05-926d-46d1-9da0-85b0e992adcf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef28329565d namespace: kserve-ci-e2e-test resourceVersion: "18873" uid: 574fa84a-a429-4150-8ac8-32542da0dcd2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef283f67138 namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 773a5d7f-9621-4da1-a66d-99389b367b6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef28403c52d namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 121caeec-6633-41d0-80e8-c44dde50dcb1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef286f307bc namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: f427a5ff-a868-4196-a47b-fcfc594df441 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:42Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef2875fe6f2 namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: c28a12c8-269e-4ae0-963f-21e963772d36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90ef2f4784312 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: 76cc6c07-d93d-4651-a4a9-3219185e8dd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90f04b9bfec82 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: b8c30b5b-d1b7-4e76-93bb-b47974fc1b56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: ea5c6196-c9a6-4e90-92bd-0fc565afcb05 kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89.18a90f04b9c09c39 namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 35bdc318-6ab2-4f92-acbd-3fc8959f4c7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d864a-predictor-7d759dc8c4 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 29d22a0c-3d68-4567-965b-9b5f59d75735 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: 'Created pod: success-200-isvc-d864a-predictor-7d759dc8c4-8pc89' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-d864a-predictor-7d759dc8c4.18a90ef20c0ae8e5 namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: 5ab99f94-bf44-4348-9584-3755fe2fab95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18773" uid: 785b5b34-0d2a-4df9-8e24-ee23201dc715 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: Scaled up replica set success-200-isvc-d864a-predictor-7d759dc8c4 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-d864a-predictor.18a90ef20a1b72b0 namespace: kserve-ci-e2e-test resourceVersion: "18779" uid: 73eca820-382d-4c38-adbd-862f214dd0ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: fcbe54d7-ada5-4a50-9b8c-9dfc5a5a616a kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-d864a-predictor.18a90ef588a23819 namespace: kserve-ci-e2e-test resourceVersion: "19170" uid: d59752e5-1e8a-48e2-b808-9a344fdeff98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: fcbe54d7-ada5-4a50-9b8c-9dfc5a5a616a kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:10Z" name: success-200-isvc-d864a-predictor.18a90ef58918b5d4 namespace: kserve-ci-e2e-test resourceVersion: "19171" uid: c35cde59-04bc-4575-8b8d-2dbebc31e8d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19016" uid: fcbe54d7-ada5-4a50-9b8c-9dfc5a5a616a kind: Event lastTimestamp: "2026-04-23T18:13:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:55Z" name: success-200-isvc-d864a-predictor.18a90efc8770071d namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: be02e55d-edc5-4e61-b493-3389d377b8ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d864a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19016" uid: fcbe54d7-ada5-4a50-9b8c-9dfc5a5a616a kind: Event lastTimestamp: "2026-04-23T18:13:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:55Z" name: success-200-isvc-d864a-predictor.18a90efc87dd28c6 namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: 10aefe3d-ec50-49f7-af2c-6a38f1edddc3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: d2a0bdc6-f9cc-4198-85c1-56ce1ac79253 kind: Event lastTimestamp: "2026-04-23T18:13:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:30Z" name: success-200-isvc-d864a.18a90ef209f56fb4 namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: aaa696b7-a6da-4f80-b660-924b0a7cdbed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: d2a0bdc6-f9cc-4198-85c1-56ce1ac79253 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-d864a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d864a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-d864a.18a90ef20aa0a12e namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 92a9cd7c-6753-4fe1-98e0-334305bef95a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: d2a0bdc6-f9cc-4198-85c1-56ce1ac79253 kind: Event lastTimestamp: "2026-04-23T18:12:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d864a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:40Z" name: success-200-isvc-d864a.18a90ef20aa0f28e namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: eab610c5-47b5-4b8a-ae6b-a7b91f34c2b6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d864a namespace: kserve-ci-e2e-test resourceVersion: "19503" uid: d2a0bdc6-f9cc-4198-85c1-56ce1ac79253 kind: Event lastTimestamp: "2026-04-23T18:13:30Z" message: InferenceService [success-200-isvc-d864a] is Ready metadata: creationTimestamp: "2026-04-23T18:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:30Z" name: success-200-isvc-d864a.18a90efdb0256e09 namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: 49fd8b94-30e6-49d2-b8b1-2e7ccebf7cb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:14:00.965907Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e1fac-predictor-b698b876d-t4s65 to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04bafecc47 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: fbd6b32f-923e-4aa0-aa8a-17011304a699 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04d4dbd1f4 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: 30a43dde-5cd8-40cf-baa9-86c27a975dd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04d63fe163 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: e5736ca3-20f9-4ecf-bcf1-f474cbbb01e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04d94106a2 namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: f98d847f-09d6-4d4a-ad12-35697d19b862 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04d9b5fe57 namespace: kserve-ci-e2e-test resourceVersion: "19914" uid: 3de17eb5-39d5-4248-8d32-f4aa0568985f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04d9c38a9e namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 41d28807-9fb9-4379-98f3-d3494581898a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04dc2f91b6 namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: c04cf5eb-504d-4d2b-b212-aef966fb62fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:14:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:01Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f04dca2b930 namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 3a09edc9-b50c-4cae-8c41-f20d52b56899 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:23:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:18Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f052ccc6737 namespace: kserve-ci-e2e-test resourceVersion: "24231" uid: 691f42e1-a917-485c-818a-3bb56919cab6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f85e825b758 namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 294e7183-e591-4351-a2ca-81e713fc1ad0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:23:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:15Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f85e8264aa2 namespace: kserve-ci-e2e-test resourceVersion: "24117" uid: cfcf1ca0-69a0-4791-ba9f-3b9e35ce635c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e1fac-predictor-b698b876d-t4s65 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 3f993db0-82b6-4d8a-ac9d-056859ab0898 kind: Event lastTimestamp: "2026-04-23T18:23:18Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:18Z" name: success-200-isvc-e1fac-predictor-b698b876d-t4s65.18a90f86a1399be8 namespace: kserve-ci-e2e-test resourceVersion: "24230" uid: 2fdaa539-055c-4f0a-923a-d96b1f439145 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e1fac-predictor-b698b876d namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: c8f4ba68-a97f-42c7-9320-8ace390b247e kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: 'Created pod: success-200-isvc-e1fac-predictor-b698b876d-t4s65' metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-e1fac-predictor-b698b876d.18a90f04ba669cb7 namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: c407b81e-4140-4fa6-ab15-2414ea2e0b50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 55ee584b-f6ff-489e-9a3f-ba24c25f2103 kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: Scaled up replica set success-200-isvc-e1fac-predictor-b698b876d from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-e1fac-predictor.18a90f04b8abb6a2 namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 851e8632-add0-4213-bb01-63a03efd3a88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:14:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: 60e34455-64d1-472a-b880-c03236f7e61b kind: Event lastTimestamp: "2026-04-23T18:14:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:30Z" name: success-200-isvc-e1fac-predictor.18a90f08378f037c namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: e3a74391-885c-49e8-b69a-29f07f8e9b72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:14:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: 60e34455-64d1-472a-b880-c03236f7e61b kind: Event lastTimestamp: "2026-04-23T18:14:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:30Z" name: success-200-isvc-e1fac-predictor.18a90f083807b2a1 namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: ac1b9aae-0e21-42a1-91a4-291bd07a7d72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 60e34455-64d1-472a-b880-c03236f7e61b kind: Event lastTimestamp: "2026-04-23T18:15:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:16Z" name: success-200-isvc-e1fac-predictor.18a90f0f35ebbf76 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 2e66e1f7-0826-41fc-82be-ad0962be98f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e1fac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 60e34455-64d1-472a-b880-c03236f7e61b kind: Event lastTimestamp: "2026-04-23T18:15:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:16Z" name: success-200-isvc-e1fac-predictor.18a90f0f365be982 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 13403f38-95cd-4abb-bb0d-fa10feb88b5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 56da962a-a733-43ab-b529-3f15e99d5b59 kind: Event lastTimestamp: "2026-04-23T18:14:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:49Z" name: success-200-isvc-e1fac.18a90f04b8b4f811 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 46930bed-6cb6-4579-9f16-15dd907e5b90 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 56da962a-a733-43ab-b529-3f15e99d5b59 kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-e1fac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e1fac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-e1fac.18a90f04b9342519 namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: 77839d77-1700-43b4-82e9-4a4d30818e56 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 56da962a-a733-43ab-b529-3f15e99d5b59 kind: Event lastTimestamp: "2026-04-23T18:14:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e1fac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:00Z" name: success-200-isvc-e1fac.18a90f04b9347892 namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: efffb972-0f86-4ab2-9dd6-355f443a9042 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e1fac namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 56da962a-a733-43ab-b529-3f15e99d5b59 kind: Event lastTimestamp: "2026-04-23T18:14:48Z" message: InferenceService [success-200-isvc-e1fac] is Ready metadata: creationTimestamp: "2026-04-23T18:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:48Z" name: success-200-isvc-e1fac.18a90f0fe8fb6d3b namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 5078fa37-12aa-4b10-ad68-261e677b17ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:01:15.743164Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12848" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k to ip-10-0-131-177.ec2.internal metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:01:15Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e52902a52e4 namespace: kserve-ci-e2e-test resourceVersion: "12853" uid: 5d83c119-09dc-4789-9097-3cb7868d3981 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12858" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:01:16Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e52a946a60b namespace: kserve-ci-e2e-test resourceVersion: "12917" uid: ae2bc32e-68fe-45ee-a269-7dfdc2ed3174 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:16Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" metadata: creationTimestamp: "2026-04-23T18:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:16Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e52aa85c85b namespace: kserve-ci-e2e-test resourceVersion: "12919" uid: b4f8811e-ef10-4e26-a47e-6e0db20fc01b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1417" in 13.073s (13.073s including waiting). Image size: 1337612401 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e55b5bceaed namespace: kserve-ci-e2e-test resourceVersion: "13080" uid: 1fe05217-b56d-4c17-b3ff-a1a255ff755d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e55b8b4ec99 namespace: kserve-ci-e2e-test resourceVersion: "13081" uid: 6c0e914f-ffc9-445a-bc1f-b00b9a675206 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e55b92bf428 namespace: kserve-ci-e2e-test resourceVersion: "13083" uid: 4d41e064-e7de-451e-add2-8ca083dff484 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:29Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e55b9383837 namespace: kserve-ci-e2e-test resourceVersion: "13084" uid: d460c916-8c0f-4a08-8c6b-5ac251a1d40b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.263s (2.263s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:31Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e56401f9afd namespace: kserve-ci-e2e-test resourceVersion: "13137" uid: d099fda2-6ca9-49aa-b423-4112ee011668 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:31Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e56435a0fbd namespace: kserve-ci-e2e-test resourceVersion: "13138" uid: ad7408b4-5478-4fd2-bafe-370fb02ba585 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:01:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:01:31Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e5643d4e9a6 namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 1dfe6347-0861-411f-89f0-e7a02578dd11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:01:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:02:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:09Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e56b061b9e6 namespace: kserve-ci-e2e-test resourceVersion: "13490" uid: 48d1c09d-76f5-4068-b0d8-75c2ef93059e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e6789b411af namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: 69216191-d20c-46d0-a0af-1b425aeacec7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 77f89529-93b4-4590-bf68-4bc1e4d717bd kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k.18a90e6789b4910e namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: bd20244d-1b17-4561-ab81-07d8f7f1e1bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-177.ec2.internal source: component: kubelet host: ip-10-0-131-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e7d75-predictor-7dcb967d8d namespace: kserve-ci-e2e-test resourceVersion: "12841" uid: f47c180f-55ff-4e06-abec-4641adbe5cee kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: 'Created pod: success-200-isvc-e7d75-predictor-7dcb967d8d-2js4k' metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: success-200-isvc-e7d75-predictor-7dcb967d8d.18a90e528faca773 namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: 04581669-7974-4dd8-bfc8-c2234a944d0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "12839" uid: 34710f7f-07a1-4d0f-baf3-b9af18b24943 kind: Event lastTimestamp: "2026-04-23T18:01:15Z" message: Scaled up replica set success-200-isvc-e7d75-predictor-7dcb967d8d from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:01:15Z" name: success-200-isvc-e7d75-predictor.18a90e528de4cb33 namespace: kserve-ci-e2e-test resourceVersion: "12843" uid: ba30105e-36a3-4593-b778-af26c6d3a813 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "12840" uid: 8aef4e03-c613-4113-b4cf-3a4673b2f196 kind: Event lastTimestamp: "2026-04-23T18:02:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:00Z" name: success-200-isvc-e7d75-predictor.18a90e560c8019e6 namespace: kserve-ci-e2e-test resourceVersion: "13420" uid: 96a2f513-e54b-486b-9e58-15cd0cc6a48f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "12840" uid: 8aef4e03-c613-4113-b4cf-3a4673b2f196 kind: Event lastTimestamp: "2026-04-23T18:02:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:00Z" name: success-200-isvc-e7d75-predictor.18a90e560ce7f866 namespace: kserve-ci-e2e-test resourceVersion: "13421" uid: 2a5638e4-6293-492c-a927-897b5d6f0cf7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "13117" uid: 8aef4e03-c613-4113-b4cf-3a4673b2f196 kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-e7d75-predictor.18a90e608971146d namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 709bb546-7f95-424c-9a11-32a6013a65ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7d75-predictor namespace: kserve-ci-e2e-test resourceVersion: "13117" uid: 8aef4e03-c613-4113-b4cf-3a4673b2f196 kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:45Z" name: success-200-isvc-e7d75-predictor.18a90e6089eaaa9a namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: f4458e48-04de-44e0-8f23-ccd2e178d6b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7d75 namespace: kserve-ci-e2e-test resourceVersion: "12832" uid: b8a38e9d-68f3-4e95-870d-30b0b88a636e kind: Event lastTimestamp: "2026-04-23T18:02:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:19Z" name: success-200-isvc-e7d75.18a90e528dc3827f namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: c7a62ab2-ae23-4f6c-95f6-7f8f462c15df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7d75 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: b8a38e9d-68f3-4e95-870d-30b0b88a636e kind: Event lastTimestamp: "2026-04-23T18:02:19Z" message: InferenceService [success-200-isvc-e7d75] is Ready metadata: creationTimestamp: "2026-04-23T18:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:19Z" name: success-200-isvc-e7d75.18a90e616b67addc namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: d20d5cf5-8738-4ca4-9e83-9cfe20e737b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:03:46.000630Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3d086-59947dfbb9-jnm97 to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:03:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:03:46Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90e758c3550d8 namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: 941b066a-3a35-4982-8f1f-20201b02c472 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: "2026-04-23T18:03:46Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:03:46Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90e75a53845dd namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 15242653-c8d0-4ca2-8b9f-15949bccc0c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d086} kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: "2026-04-23T18:03:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:46Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90e75a691b706 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: c088ee4f-749c-402a-8524-c7ce0d852004 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d086} kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: "2026-04-23T18:03:46Z" message: 'Created container: switch-graph-3d086' metadata: creationTimestamp: "2026-04-23T18:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:46Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90e75a972da4b namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 6c456c47-912e-4fd6-94be-8ca3e17e3445 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d086} kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: "2026-04-23T18:03:46Z" message: Started container switch-graph-3d086 metadata: creationTimestamp: "2026-04-23T18:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:46Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90e75a9ea7068 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 237c1ea9-a220-4fec-a0d8-173448ae0c04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d086} kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: "2026-04-23T18:12:00Z" message: Stopping container switch-graph-3d086 metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:00Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90ee8b24fee11 namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: 005649cb-f8fa-4c6b-ad76-b1273bd0b175 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d086} kind: Pod name: switch-graph-3d086-59947dfbb9-jnm97 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 8e72973c-b887-4181-b2d5-936ddc1b927f kind: Event lastTimestamp: "2026-04-23T18:12:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:28Z" name: switch-graph-3d086-59947dfbb9-jnm97.18a90ee95366ee38 namespace: kserve-ci-e2e-test resourceVersion: "18639" uid: 1f914656-64a3-4758-8138-d308f1591886 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3d086-59947dfbb9 namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 73d0282c-8d2a-49af-a579-4aeab4027eff kind: Event lastTimestamp: "2026-04-23T18:03:45Z" message: 'Created pod: switch-graph-3d086-59947dfbb9-jnm97' metadata: creationTimestamp: "2026-04-23T18:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:45Z" name: switch-graph-3d086-59947dfbb9.18a90e758b9391bb namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: c203d845-fa79-4399-b195-99c2821fead0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 664f0d3b-e75f-419b-aba8-e4e04519758d kind: Event lastTimestamp: "2026-04-23T18:03:45Z" message: Scaled up replica set switch-graph-3d086-59947dfbb9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:45Z" name: switch-graph-3d086.18a90e758a332460 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 525e6cd4-315a-4ff4-a35a-115cfb5c8682 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: dbcf28c2-ff46-43ab-9503-18c7cfe184fb kind: Event lastTimestamp: "2026-04-23T18:03:46Z" message: InferenceGraph [switch-graph-3d086] is Ready metadata: creationTimestamp: "2026-04-23T18:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:46Z" name: switch-graph-3d086.18a90e7590357742 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: db117683-e978-4dd3-9388-dd93e35af113 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 968b4bcf-fd0f-4f7d-8eb0-95972e9ea276 kind: Event lastTimestamp: "2026-04-23T18:04:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:00Z" name: switch-graph-3d086.18a90e7908b3be87 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 19f8e2d2-7e52-478a-be20-6eaf25d288eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 968b4bcf-fd0f-4f7d-8eb0-95972e9ea276 kind: Event lastTimestamp: "2026-04-23T18:04:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:00Z" name: switch-graph-3d086.18a90e7909255ae6 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: e5b1d586-8e43-4a2c-859a-1e87b886f56c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 968b4bcf-fd0f-4f7d-8eb0-95972e9ea276 kind: Event lastTimestamp: "2026-04-23T18:04:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:15Z" name: switch-graph-3d086.18a90e7c880f36ce namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 63905cb8-1f58-4b0d-a56d-cc2f9ce81aad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d086 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 968b4bcf-fd0f-4f7d-8eb0-95972e9ea276 kind: Event lastTimestamp: "2026-04-23T18:04:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:16Z" name: switch-graph-3d086.18a90e7c88d8b8c0 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 39f3c3ae-5dc2-41a5-a9fd-b9404a88a5ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:24:15.888784Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24917" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-87aee-db68b6cb9-f6x9v to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:24:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:24:15Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a90f93e7413eb1 namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 3450339f-5195-480e-892d-536c79a3d69d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:24:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-87aee-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:16Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a90f93f2ecdd80 namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: 07662519-fd3a-49cd-b8ce-07811416553a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:24:16Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:24:16Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a90f9423ed3592 namespace: kserve-ci-e2e-test resourceVersion: "24948" uid: 805748c2-a7db-4d71-8bd5-e84c2bb42c4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87aee} kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:24:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:16Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a90f9425437e13 namespace: kserve-ci-e2e-test resourceVersion: "24950" uid: 0e3b87fd-7027-4d85-b383-5a4faf7ba975 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87aee} kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:24:16Z" message: 'Created container: switch-graph-87aee' metadata: creationTimestamp: "2026-04-23T18:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:16Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a90f9427fee5ee namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 21445595-6722-4080-942c-3b5b15c81e4f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87aee} kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:24:16Z" message: Started container switch-graph-87aee metadata: creationTimestamp: "2026-04-23T18:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:16Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a90f9428715447 namespace: kserve-ci-e2e-test resourceVersion: "24952" uid: 53b3634d-458d-4d86-85c2-1b4329c20345 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87aee} kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:40:35Z" message: Stopping container switch-graph-87aee metadata: creationTimestamp: "2026-04-23T18:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:35Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a91077e2212f4b namespace: kserve-ci-e2e-test resourceVersion: "32109" uid: 19e470fb-9980-4f6b-b60a-b7d9b1a95ae5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:40:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87aee} kind: Pod name: switch-graph-87aee-db68b6cb9-f6x9v namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 8d08ae5c-7408-451d-b12c-4c9ec3720cd0 kind: Event lastTimestamp: "2026-04-23T18:40:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:40:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:40:58Z" name: switch-graph-87aee-db68b6cb9-f6x9v.18a91078b1b81915 namespace: kserve-ci-e2e-test resourceVersion: "32352" uid: e32d9a6a-9866-4044-b90e-1d3a20726aff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-87aee-db68b6cb9 namespace: kserve-ci-e2e-test resourceVersion: "24914" uid: a9e09058-9633-4d3c-946a-edcbe41e0261 kind: Event lastTimestamp: "2026-04-23T18:24:15Z" message: 'Created pod: switch-graph-87aee-db68b6cb9-f6x9v' metadata: creationTimestamp: "2026-04-23T18:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:15Z" name: switch-graph-87aee-db68b6cb9.18a90f93e6caa5a3 namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 0b94e453-b8eb-49d9-b3ef-d8a7ac05e94e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-87aee namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: e918601e-a683-4115-a565-7a0ac1891dd6 kind: Event lastTimestamp: "2026-04-23T18:24:15Z" message: Scaled up replica set switch-graph-87aee-db68b6cb9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:15Z" name: switch-graph-87aee.18a90f93e5e56dfe namespace: kserve-ci-e2e-test resourceVersion: "24915" uid: 50ae1189-0ec4-4760-9862-ef1adea40fe9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-87aee namespace: kserve-ci-e2e-test resourceVersion: "24932" uid: 0d6310c4-db63-4f08-87de-5cd95f2b4813 kind: Event lastTimestamp: "2026-04-23T18:24:15Z" message: InferenceGraph [switch-graph-87aee] is Ready metadata: creationTimestamp: "2026-04-23T18:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:15Z" name: switch-graph-87aee.18a90f93ec5a637f namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 053a98ec-02dc-403d-b0c8-dbd8c1c41df6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87aee namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: fa7ba433-7463-4959-8d0e-f7584610a7e7 kind: Event lastTimestamp: "2026-04-23T18:24:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:45Z" name: switch-graph-87aee.18a90f9764585f99 namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: b177bf8c-0322-4a5a-96a4-909e943cf145 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87aee namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: fa7ba433-7463-4959-8d0e-f7584610a7e7 kind: Event lastTimestamp: "2026-04-23T18:24:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:45Z" name: switch-graph-87aee.18a90f9764bd6eee namespace: kserve-ci-e2e-test resourceVersion: "25235" uid: 96a069c1-dfaa-4b9c-9537-4a308873a6bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:02:35.540664Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e7d75-776949f8dd-kqj2m to ip-10-0-133-178.ec2.internal metadata: creationTimestamp: "2026-04-23T18:02:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:02:35Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e6524778e53 namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: a59fcd24-7bd0-475d-b373-08cae430c858 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b646fbdfd-4gsmg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:02:35Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:02:35Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e653d5e7d89 namespace: kserve-ci-e2e-test resourceVersion: "13754" uid: 7908c1e2-7358-4577-bcfe-1147e359745b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7d75} kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:02:35Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" metadata: creationTimestamp: "2026-04-23T18:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:35Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e653e9ee3fe namespace: kserve-ci-e2e-test resourceVersion: "13756" uid: 272b6c19-9f16-478f-8fe5-67968c90afdc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7d75} kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:02:38Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" in 2.426s (2.426s including waiting). Image size: 216346204 bytes.' metadata: creationTimestamp: "2026-04-23T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:38Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e65cf44e1cb namespace: kserve-ci-e2e-test resourceVersion: "13772" uid: f2f6485e-d229-40ab-ac2b-6c3d98b29616 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7d75} kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:02:38Z" message: 'Created container: switch-graph-e7d75' metadata: creationTimestamp: "2026-04-23T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:38Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e65d22f92df namespace: kserve-ci-e2e-test resourceVersion: "13773" uid: 7c67f8e0-46cc-4584-895c-272bf918ba37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7d75} kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:02:38Z" message: Started container switch-graph-e7d75 metadata: creationTimestamp: "2026-04-23T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:38Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e65d29c7e74 namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 84000c1b-58aa-4199-80e1-871e44bd4fb4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7d75} kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Stopping container switch-graph-e7d75 metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e67833968c8 namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: cbddec6c-5dac-4d9a-80de-449dca8e0494 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:02:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7d75} kind: Pod name: switch-graph-e7d75-776949f8dd-kqj2m namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 413f510b-b6a9-44b1-8428-3a0e318ea683 kind: Event lastTimestamp: "2026-04-23T18:03:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:15Z" name: switch-graph-e7d75-776949f8dd-kqj2m.18a90e68862f2271 namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: 4f4a79f9-4ef6-4918-9e68-0e29e8c97cb6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-178.ec2.internal source: component: kubelet host: ip-10-0-133-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e7d75-776949f8dd namespace: kserve-ci-e2e-test resourceVersion: "13729" uid: 49cb5c84-478e-4b02-bb95-c643dd232c1f kind: Event lastTimestamp: "2026-04-23T18:02:35Z" message: 'Created pod: switch-graph-e7d75-776949f8dd-kqj2m' metadata: creationTimestamp: "2026-04-23T18:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:35Z" name: switch-graph-e7d75-776949f8dd.18a90e65240ddbe0 namespace: kserve-ci-e2e-test resourceVersion: "13733" uid: f594170c-b2be-4ddd-b82f-4f208dfb25fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e7d75 namespace: kserve-ci-e2e-test resourceVersion: "13727" uid: 2af09ad6-4317-4b85-84cc-db90083145c5 kind: Event lastTimestamp: "2026-04-23T18:02:35Z" message: Scaled up replica set switch-graph-e7d75-776949f8dd from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:35Z" name: switch-graph-e7d75.18a90e65235b1f1d namespace: kserve-ci-e2e-test resourceVersion: "13730" uid: 6b40e661-d0a8-4478-a5cb-105f5ff3bb01 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e7d75 namespace: kserve-ci-e2e-test resourceVersion: "13747" uid: 6d81d1f4-eabc-4bd2-a4f6-1dc6cd45528a kind: Event lastTimestamp: "2026-04-23T18:02:35Z" message: InferenceGraph [switch-graph-e7d75] is Ready metadata: creationTimestamp: "2026-04-23T18:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:35Z" name: switch-graph-e7d75.18a90e652784c7f5 namespace: kserve-ci-e2e-test resourceVersion: "13749" uid: 8c466508-f3be-44f5-bc7e-3686f42680e0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "32354"