--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:49:00.028195Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b12e8-db4cf88d6-fzbjk to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:49:00Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd06fe056ca2 namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 2ca1fd53-28a2-4bc5-b740-07c52b379268 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-b12e8-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd070829e892 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: 30acf4fa-5f9f-4d95-9da2-0c6fa2aa9c15 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:01Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:49:01Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd073c4362b5 namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: ba2ba729-1b34-45b5-a409-e7b8277de149 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b12e8} kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:01Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd073db53ab6 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: e9cb0e01-b73f-4871-8b30-4a2d17c2149f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b12e8} kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:01Z" message: 'Created container: ensemble-graph-b12e8' metadata: creationTimestamp: "2026-07-29T10:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:01Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd0740af1ee8 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 6c457310-5b1f-45fd-957e-fa73c8b73e3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b12e8} kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:01Z" message: Started container ensemble-graph-b12e8 metadata: creationTimestamp: "2026-07-29T10:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:01Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd0741298da3 namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 42813a72-5f41-47ca-a6f1-36113b0eaa4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b12e8} kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Stopping container ensemble-graph-b12e8 metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd095678ca4b namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: c50530d4-b9bc-48d6-a4d3-812d48606f22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b12e8} kind: Pod name: ensemble-graph-b12e8-db4cf88d6-fzbjk namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 8416c8b6-3d27-4e6b-8a17-e106ec8aa4c3 kind: Event lastTimestamp: "2026-07-29T10:49:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:37Z" name: ensemble-graph-b12e8-db4cf88d6-fzbjk.18c6bd0a024e4b70 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 7804b675-4551-4c88-86eb-d55d701b2a15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b12e8-db4cf88d6 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 9d3c8f25-7f7d-414d-a68b-45c6e9f883ff kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'Created pod: ensemble-graph-b12e8-db4cf88d6-fzbjk' metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:00Z" name: ensemble-graph-b12e8-db4cf88d6.18c6bd06fd95883f namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 40c6cc9d-b15d-4762-a24f-884d8b1c3a9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: a70fdfd4-13fc-4bcf-9b32-d7d74f98e1e4 kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Scaled up replica set ensemble-graph-b12e8-db4cf88d6 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:00Z" name: ensemble-graph-b12e8.18c6bd06fcebbe9b namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 94f4849c-50e9-439e-8b3d-cf127ccff5fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: 01ea9113-26d4-4392-9362-debdd0dc4c7e kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: InferenceGraph [ensemble-graph-b12e8] is Ready metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:00Z" name: ensemble-graph-b12e8.18c6bd0701df630a namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: a1a922b6-2008-4521-a923-3dca8b0c072f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:50:10.306674Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-d2d23-557d84f99-bwkd2 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:50:10Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd175af1ed44 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: d96d6124-661c-4238-b805-11315df54da1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-d2d23-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd1769a8c4fb namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 58fbc886-749f-45e5-ad3b-b68f66a54ab3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:50:11Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd17999e3d5b namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: 11fb9d4b-80d8-46ff-b693-524eb7ccc29d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d2d23} kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd179afc6b2b namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: 24cfc193-c960-4caf-9c63-9f7b63a4dcfe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d2d23} kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: 'Created container: ensemble-graph-d2d23' metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd179efc021b namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: bbd7ff98-9d12-4110-9548-c7dfc0113687 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d2d23} kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Started container ensemble-graph-d2d23 metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd179f7a56ee namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 5395f10e-46fa-4857-adba-2ae5446ce045 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d2d23} kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:58:24Z" message: Stopping container ensemble-graph-d2d23 metadata: creationTimestamp: "2026-07-29T10:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:24Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd8a803abe85 namespace: kserve-ci-e2e-test resourceVersion: "24978" uid: 07358cfa-2b3b-4876-980b-ddccb9c728bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d2d23} kind: Pod name: ensemble-graph-d2d23-557d84f99-bwkd2 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f8bc01a7-7322-4593-92ff-5f0e5e9a4e12 kind: Event lastTimestamp: "2026-07-29T10:58:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:53Z" name: ensemble-graph-d2d23-557d84f99-bwkd2.18c6bd8b4c6ad47b namespace: kserve-ci-e2e-test resourceVersion: "25315" uid: da309514-1066-4dd6-8aaa-3bf6c0d7a688 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-d2d23-557d84f99 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: e60d1934-389c-4dc2-9865-4552172d1f9c kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'Created pod: ensemble-graph-d2d23-557d84f99-bwkd2' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:10Z" name: ensemble-graph-d2d23-557d84f99.18c6bd175a586b27 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 0302461d-89a7-48e2-aa2a-5ccbc166391d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: b3fccf83-09aa-419e-825b-928729c7d490 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Scaled up replica set ensemble-graph-d2d23-557d84f99 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:10Z" name: ensemble-graph-d2d23.18c6bd17599bfd70 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: a12ed6fe-7510-43a3-81ff-2498a84dd877 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: f6a25d6a-6e9f-4083-8e2d-3be9cc15e193 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: InferenceGraph [ensemble-graph-d2d23] is Ready metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:10Z" name: ensemble-graph-d2d23.18c6bd1761e70fbf namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 3279dac0-7546-4436-af1f-26ec5348ed28 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:50:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 442eb2ba-57c1-4b95-9b56-406c228975ce kind: Event lastTimestamp: "2026-07-29T10:50:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:40Z" name: ensemble-graph-d2d23.18c6bd1ad8150873 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: b31551b1-cf11-4d51-857f-3314624a3b12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:50:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 442eb2ba-57c1-4b95-9b56-406c228975ce kind: Event lastTimestamp: "2026-07-29T10:50:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:40Z" name: ensemble-graph-d2d23.18c6bd1ad87c998f namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: aac5c70a-3731-4089-8e1c-55710bcf289f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:50:10.143928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0934b-predictor-5949bbfc75-kftfs to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd17513e9dac namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 2a1c4284-381f-4855-a534-c21d968c74a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd176b894a73 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 007c8047-dc81-463b-b82e-cf7260c32471 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd176d147ab1 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: d3838bc9-0ba1-4892-ac37-a5e3dd08048a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd1770480403 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 622d5411-1077-4889-9682-23f0ab5015bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd1770cf843c namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: ce5d2613-e302-43ba-bbed-127c8b642e9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd1770d94d33 namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: f982d6ff-966a-4abb-bf4c-8e1234c95aad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd1773b2628d namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 27ba5d34-2b80-426e-bc9a-db28f4c9afb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd1774299aa7 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 443e8834-e73c-4523-b0da-c95f71c863b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:59:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:27Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd178a90de02 namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: 015d9597-2506-407e-bbd4-f9c0fcee4e3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:24Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd987d98ec1c namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: f78ce0e3-f895-4dd6-9651-a13251f44053 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:24Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd987d998086 namespace: kserve-ci-e2e-test resourceVersion: "25639" uid: cdb0ebab-46f4-44a6-a5e2-539c71a3b62a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ee8afd5e-7b5f-4b67-b55b-fb8f8dfccbfc kind: Event lastTimestamp: "2026-07-29T10:59:27Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:27Z" name: error-404-isvc-0934b-predictor-5949bbfc75-kftfs.18c6bd98ff271cb8 namespace: kserve-ci-e2e-test resourceVersion: "25762" uid: fdd1b42b-f9e5-4415-9f33-4441286a7ade reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0934b-predictor-5949bbfc75 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 8f3cf273-4926-42d1-a091-564f2dfe7e84 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'Created pod: error-404-isvc-0934b-predictor-5949bbfc75-kftfs' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor-5949bbfc75.18c6bd1750dcb4bc namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 2433c325-b68a-435b-b6d4-3ec384f48ccb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 462ca918-3f8f-4ec1-bfc8-30127e179143 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Scaled up replica set error-404-isvc-0934b-predictor-5949bbfc75 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b-predictor.18c6bd174f33ea54 namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: e3f93e48-f843-4d6e-a904-cce694b62faf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:50:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 652d3f3a-e8e7-41ea-af3b-59567e510965 kind: Event lastTimestamp: "2026-07-29T10:50:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:55Z" name: error-404-isvc-0934b-predictor.18c6bd1acdb9a104 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: df9ec457-240e-4fff-938f-a6d5060d4cbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:50:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 652d3f3a-e8e7-41ea-af3b-59567e510965 kind: Event lastTimestamp: "2026-07-29T10:50:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:55Z" name: error-404-isvc-0934b-predictor.18c6bd1ace23319e namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: a6d14134-ea4f-44c0-944c-193494fcd725 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:50:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 652d3f3a-e8e7-41ea-af3b-59567e510965 kind: Event lastTimestamp: "2026-07-29T10:51:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:40Z" name: error-404-isvc-0934b-predictor.18c6bd1e4cda5443 namespace: kserve-ci-e2e-test resourceVersion: "22217" uid: 667b0ee6-2d00-46d8-a8d1-4a17190e68cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:50:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 652d3f3a-e8e7-41ea-af3b-59567e510965 kind: Event lastTimestamp: "2026-07-29T10:51:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:40Z" name: error-404-isvc-0934b-predictor.18c6bd1e4d4305de namespace: kserve-ci-e2e-test resourceVersion: "22218" uid: 188b1d6f-a231-4221-ae03-f457e86b22ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0934b namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: b7b46f61-9b99-4e96-ba7b-e7a483b8e960 kind: Event lastTimestamp: "2026-07-29T10:50:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:57Z" name: error-404-isvc-0934b.18c6bd174f332b00 namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 8fc054c2-db9e-43e3-81f0-c4c04dbda406 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0934b namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: b7b46f61-9b99-4e96-ba7b-e7a483b8e960 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-0934b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0934b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b.18c6bd174fec31dc namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 8a3c71c1-9b66-40b9-baef-40c166ef99fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0934b namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: b7b46f61-9b99-4e96-ba7b-e7a483b8e960 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0934b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-0934b.18c6bd174fec94a7 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: fd27b1af-1c26-4803-ad45-dd63400ca927 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0934b namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: b7b46f61-9b99-4e96-ba7b-e7a483b8e960 kind: Event lastTimestamp: "2026-07-29T10:50:57Z" message: InferenceService [error-404-isvc-0934b] is Ready metadata: creationTimestamp: "2026-07-29T10:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:57Z" name: error-404-isvc-0934b.18c6bd22458113c4 namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: 495cb3dd-4bde-4b65-8583-50647970d141 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:38:45.092105Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0a743-predictor-589d4b48df-5ss6c to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:38:45Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc77d0f950c5 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: a834da09-ca0f-4a3d-986b-5c283fa060f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0a743-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc77dd48b4b1 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 20b0678b-9455-4e88-81f3-ea6402318b89 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc780eb9d5c7 namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 0176dfa5-3a14-446f-973e-6d16c8befe54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc78104df22c namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: 1b636039-385c-4290-9c93-c13ea2561ed8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc781385befc namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 7fbf0a85-f19a-4fc3-9a6d-f78f66d6d48c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc7813f97439 namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: 57a4ba67-cfd0-420c-81c7-b5c6d32450db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc781403e61c namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: c74a46f8-bcfc-44a7-a175-4d352064025f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc7816fe1d7b namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: f08660f0-ec08-4ad6-9df9-aa7cf157c755 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:38:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:46Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc7817c386b2 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: b58a8cfd-e0b0-4bc5-a412-cf98e469fa71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:48:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:02Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bc7831f9527f namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 91a761fd-beb7-4399-9065-a1f6fade3abe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bcf909a95fa2 namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: b44b3afc-e13c-4291-8cf2-7d1f5468a1ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bcf909a9d6be namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: ee741487-56ce-4591-9cdd-8b7758120af8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 03c31e5c-3e05-49ea-86e1-c218f4dd031f kind: Event lastTimestamp: "2026-07-29T10:48:02Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:02Z" name: error-404-isvc-0a743-predictor-589d4b48df-5ss6c.18c6bcf9a66ea701 namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 533b228f-3e68-41cb-9ecd-4ea0b8dabf42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0a743-predictor-589d4b48df namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 42e23048-a7e7-491b-a66c-09c339ff587f kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: 'Created pod: error-404-isvc-0a743-predictor-589d4b48df-5ss6c' metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:45Z" name: error-404-isvc-0a743-predictor-589d4b48df.18c6bc77d06fc5a8 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: b8b54f40-ce28-497f-ba20-529607a217a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 133cb4fb-7e2e-4263-8b80-ed0ce63b2ed6 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: Scaled up replica set error-404-isvc-0a743-predictor-589d4b48df from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:45Z" name: error-404-isvc-0a743-predictor.18c6bc77ceb7d117 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 7c6df437-c66f-4278-8770-ca86c912dafc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:39:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: f0734672-c940-45ce-8c48-955ad01fbda6 kind: Event lastTimestamp: "2026-07-29T10:40:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:00Z" name: error-404-isvc-0a743-predictor.18c6bc7b4d30f7b7 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: bbdbb82c-ac3b-4698-b4bb-76efcb3a1f4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:39:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: f0734672-c940-45ce-8c48-955ad01fbda6 kind: Event lastTimestamp: "2026-07-29T10:40:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:00Z" name: error-404-isvc-0a743-predictor.18c6bc7b4d9887be namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 48181fde-ed7d-4daf-90ed-840e699c6687 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: f0734672-c940-45ce-8c48-955ad01fbda6 kind: Event lastTimestamp: "2026-07-29T10:39:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:15Z" name: error-404-isvc-0a743-predictor.18c6bc7ecc4e27ab namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 862b18a5-c4ee-4be0-984c-7709c46777c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: f0734672-c940-45ce-8c48-955ad01fbda6 kind: Event lastTimestamp: "2026-07-29T10:39:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:15Z" name: error-404-isvc-0a743-predictor.18c6bc7eccbe9d5d namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 4420e289-7b9d-45cf-887f-8221ee363c2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 660182b3-24d4-434d-b38b-22a905a84e2d kind: Event lastTimestamp: "2026-07-29T10:39:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:32Z" name: error-404-isvc-0a743.18c6bc77ceb863a6 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 18a9d0da-c663-4a8e-a861-38c17d2ee665 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 660182b3-24d4-434d-b38b-22a905a84e2d kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-0a743": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0a743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:45Z" name: error-404-isvc-0a743.18c6bc77cf0aae44 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 8b335e82-7b01-4e65-9f64-6301c2f34011 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 660182b3-24d4-434d-b38b-22a905a84e2d kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0a743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:45Z" name: error-404-isvc-0a743.18c6bc77cf0b0d85 namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 9b592888-7962-49c9-9d14-e52c0445599b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 660182b3-24d4-434d-b38b-22a905a84e2d kind: Event lastTimestamp: "2026-07-29T10:39:32Z" message: InferenceService [error-404-isvc-0a743] is Ready metadata: creationTimestamp: "2026-07-29T10:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:32Z" name: error-404-isvc-0a743.18c6bc82edc1a583 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 8c242482-8a8e-4ca7-b297-0f26b36311a9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:48:59.877762Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd06f50e09ab namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 2010fdcd-0ef3-4b42-a824-9f3da1f8f723 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd070f96bff9 namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: 3ed556a2-6cbb-440d-a80b-4a8cd49d981c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd0710f6e4ef namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: fdb8686b-fc0a-4ea3-abc1-31a02d316fe3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd07147ab681 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 9e39c71e-1b7a-4c96-8261-26009350ebd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd0714f70701 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: f7e56866-a428-4355-8c6f-9143e98e2460 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd071502006d namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: cecacd6c-1fc4-4806-b75d-cf5dc8f41a5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd0718182414 namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 2b7f339b-9bbf-4e83-949d-7c13a8d32b93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd0718924de2 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 00b85a24-4573-48e1-8cfe-3cf0315d5034 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:49:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:37Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd076412ea17 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: d27c4282-4247-4394-a7cb-78c8bcb09c4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd174c7b923c namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: eef76910-8dbc-4927-86d9-44b479787dcd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd174c7c8822 namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 2ce238d3-230d-4f49-bfc2-dbfb945ed108 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 599d1b04-e12c-469f-abd7-05967a2b1f8f kind: Event lastTimestamp: "2026-07-29T10:50:12Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:12Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr.18c6bd17ec443f7c namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: 15ff159b-e444-4d92-8f1d-7609179b830b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6242a-predictor-6d94cbd9cc namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: da909d32-2378-4779-a6d9-6ab5f6a1d519 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'Created pod: error-404-isvc-6242a-predictor-6d94cbd9cc-4qrtr' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-6242a-predictor-6d94cbd9cc.18c6bd06f4a46098 namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: 4e83198b-deac-41cd-8c99-bdc38a35c32c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: c56d849f-ad14-4cb6-963d-6fd64a9544aa kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Scaled up replica set error-404-isvc-6242a-predictor-6d94cbd9cc from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-6242a-predictor.18c6bd06f2ba9f09 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 0b96173d-e0ef-48fa-a3d5-080c7c394a1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 39942c2b-d55d-4a87-8829-a035b2f72b21 kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:59Z" name: error-404-isvc-6242a-predictor.18c6bd0a7144302d namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 4f4425bf-cf74-4420-97c7-befa7a19fe24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 39942c2b-d55d-4a87-8829-a035b2f72b21 kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:59Z" name: error-404-isvc-6242a-predictor.18c6bd0a71b80c66 namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: 59093ab3-1370-48ea-9389-9f113db56c97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 39942c2b-d55d-4a87-8829-a035b2f72b21 kind: Event lastTimestamp: "2026-07-29T10:49:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:29Z" name: error-404-isvc-6242a-predictor.18c6bd0df07a1040 namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: ac03df35-7292-451d-baad-a6f4aaebec0a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 39942c2b-d55d-4a87-8829-a035b2f72b21 kind: Event lastTimestamp: "2026-07-29T10:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:29Z" name: error-404-isvc-6242a-predictor.18c6bd0df0e6ed39 namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: d7bac130-92e6-4790-aae7-a09b84543846 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 89ab2777-4cc1-486a-81eb-5b852fe229a8 kind: Event lastTimestamp: "2026-07-29T10:49:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:47Z" name: error-404-isvc-6242a.18c6bd06f2be1ce5 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: ee115e3c-dae3-4144-8247-870f9ef24194 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 89ab2777-4cc1-486a-81eb-5b852fe229a8 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-6242a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6242a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-6242a.18c6bd06f317a10f namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: a8dfc38b-2168-4066-a067-9f7645232dee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 89ab2777-4cc1-486a-81eb-5b852fe229a8 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6242a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-6242a.18c6bd06f3181d9b namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: 0a74332e-dee2-483f-96e7-97ee51d7918f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 89ab2777-4cc1-486a-81eb-5b852fe229a8 kind: Event lastTimestamp: "2026-07-29T10:49:47Z" message: InferenceService [error-404-isvc-6242a] is Ready metadata: creationTimestamp: "2026-07-29T10:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:47Z" name: error-404-isvc-6242a.18c6bd121f6713e0 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 63678fd4-f269-4497-a80e-af3ffcd45e8a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:59:25.027190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-965a2-predictor-6df948c77-gjp88 to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd9882dd4a3c namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: 23c8d80a-0f90-4286-b352-b6c93caeafb5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-965a2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd988fa82f19 namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: 25c62007-54f3-4052-8518-3d4b45da3bc0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98c11b59df namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: 2e3c092c-f8a5-41df-85a8-6bcf82d8079a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98c27c717a namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: d42e1cf7-5911-43a6-94af-a2e044be202f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98c68c69f0 namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: 36078f61-1b94-45f9-8547-b2dcb3dca932 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98c70660cb namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 639aaa90-dcc4-45e0-b092-1943902a2513 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98c711f13a namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 370a29c4-09fa-4db1-a91a-ad6471f07e53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98ca5fa5a4 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 77099074-d335-489f-b606-1198fb75fc03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98cadc35f2 namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: bb87bb27-23de-4281-91b9-7262902cebe5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T11:00:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:12Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6bd98ee694150 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 054d1cca-f145-4c3e-a2c4-28850116e70a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T11:16:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T11:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:16:44Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6be8a8e11d91d namespace: kserve-ci-e2e-test resourceVersion: "33634" uid: f515cef8-7b35-4ad3-86fb-13ca2d11bb30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T11:16:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T11:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:16:44Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6be8a8e125e71 namespace: kserve-ci-e2e-test resourceVersion: "33635" uid: f6b9f67c-e08f-4114-b45e-c2ef27da46c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-965a2-predictor-6df948c77-gjp88 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 83a6e9dc-18be-4c29-a7be-abccce72a78c kind: Event lastTimestamp: "2026-07-29T11:16:47Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T11:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:16:47Z" name: error-404-isvc-965a2-predictor-6df948c77-gjp88.18c6be8b4f191cb1 namespace: kserve-ci-e2e-test resourceVersion: "33697" uid: 34e4582f-0e21-4d38-bad9-09b8c1428804 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-965a2-predictor-6df948c77 namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: cf7e4809-ecbf-4228-a55c-bad5bcac3402 kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'Created pod: error-404-isvc-965a2-predictor-6df948c77-gjp88' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-965a2-predictor-6df948c77.18c6bd98825d25a4 namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: e417b46f-129b-4dce-b930-489cdfe67444 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: dbf74f28-55eb-49ce-9272-f82666239fc1 kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Scaled up replica set error-404-isvc-965a2-predictor-6df948c77 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:24Z" name: error-404-isvc-965a2-predictor.18c6bd9880d1cef4 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: bff4344e-07cd-450c-87dd-1eb0593a2460 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: 0bac4891-579b-484e-aa72-79013d4d37a1 kind: Event lastTimestamp: "2026-07-29T10:59:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:40Z" name: error-404-isvc-965a2-predictor.18c6bd9bff56101d namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: f6ccd3a2-cbd7-4c36-a100-a89de4c06253 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: 0bac4891-579b-484e-aa72-79013d4d37a1 kind: Event lastTimestamp: "2026-07-29T10:59:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:40Z" name: error-404-isvc-965a2-predictor.18c6bd9bffe33285 namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: a81a9ff6-d460-41b8-a100-e106ce481f08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:59:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 0bac4891-579b-484e-aa72-79013d4d37a1 kind: Event lastTimestamp: "2026-07-29T11:00:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:55Z" name: error-404-isvc-965a2-predictor.18c6bd9f7e9a36f4 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 24394b32-7412-4960-8a63-0f9f8566624b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:59:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 0bac4891-579b-484e-aa72-79013d4d37a1 kind: Event lastTimestamp: "2026-07-29T11:00:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:55Z" name: error-404-isvc-965a2-predictor.18c6bd9f7f0aae40 namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: 42578159-8031-4449-9eb7-68a7a5b39d58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-965a2 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: b12c70a4-5306-494d-af09-0697571e43ae kind: Event lastTimestamp: "2026-07-29T11:00:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:22Z" name: error-404-isvc-965a2.18c6bd9880e3a719 namespace: kserve-ci-e2e-test resourceVersion: "26692" uid: ad1b19df-c966-43b9-bcc2-c0bf0eaa9106 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-965a2 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: b12c70a4-5306-494d-af09-0697571e43ae kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-965a2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-965a2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-965a2.18c6bd98817c6a46 namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: b019e1c1-ae80-48c9-8602-d416b3e80faf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-965a2 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: b12c70a4-5306-494d-af09-0697571e43ae kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-965a2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-965a2.18c6bd98817cb917 namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: 507b8e91-72f7-4faf-9183-899d8c77878c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-965a2 namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: b12c70a4-5306-494d-af09-0697571e43ae kind: Event lastTimestamp: "2026-07-29T11:00:22Z" message: InferenceService [error-404-isvc-965a2] is Ready metadata: creationTimestamp: "2026-07-29T11:00:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:22Z" name: error-404-isvc-965a2.18c6bda5fd5268e5 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 24f36d87-7934-4d89-ba02-4e24ac83d802 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:59:35.446157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-afa2d-predictor-75594848c5-cc7sx to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9aefe21c7f namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: 3e3fb149-e874-46e0-b803-e4bbba053578 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b09ff6619 namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 80b1429f-94ff-4e56-a601-84180e7eb2b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b0b76a8b6 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 7d038f73-da3a-4b4a-af2a-7ee776501954 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b0e929733 namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: c30bc83f-f071-4e21-8a79-7383270ed97f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b0f056884 namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: 493d23a8-89c5-4534-a996-93d43a321c6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b0f0ed29b namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 466997f9-3217-40d2-974f-90f1f00adb85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b11cb1991 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: e70b16ef-2ab8-4f31-b7bc-c3f8cd393305 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b12461558 namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 32bb16a7-8607-4cc8-9a1e-ce5c4ade8f43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T11:00:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:13Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6bd9b82191eec namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: 3c569b2b-34fd-45ff-bf0d-10aaee9a01bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T11:08:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T11:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:08:50Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6be1c14b698c9 namespace: kserve-ci-e2e-test resourceVersion: "30343" uid: 15a0c6ff-4d2a-4aa5-887b-bbfc27ae4ad7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 26b8799d-090d-443c-a60c-ee2419adef4e kind: Event lastTimestamp: "2026-07-29T11:08:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T11:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:08:50Z" name: error-404-isvc-afa2d-predictor-75594848c5-cc7sx.18c6be1c14b6fd70 namespace: kserve-ci-e2e-test resourceVersion: "30342" uid: b4a80efb-9843-46cc-81b1-14b3a76dd3d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-afa2d-predictor-75594848c5 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 42b939f2-c701-4c77-81d9-85244ce432ef kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'Created pod: error-404-isvc-afa2d-predictor-75594848c5-cc7sx' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor-75594848c5.18c6bd9aef7dd6e4 namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: 64f92efc-2710-4654-9a0b-3376dcff0c00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 7659e2a2-6c6b-4fd9-a0f7-46e43404ca47 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Scaled up replica set error-404-isvc-afa2d-predictor-75594848c5 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d-predictor.18c6bd9aede621b8 namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: cd8e42c4-deb8-4b85-8728-df5b2a46c22c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: fc1c25f1-4609-49fd-9424-2fa78168573f kind: Event lastTimestamp: "2026-07-29T11:00:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:05Z" name: error-404-isvc-afa2d-predictor.18c6bd9e6c52b242 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: d99fc187-6f82-4b13-8d53-7ae9974e9ad4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: fc1c25f1-4609-49fd-9424-2fa78168573f kind: Event lastTimestamp: "2026-07-29T11:00:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:05Z" name: error-404-isvc-afa2d-predictor.18c6bd9e6cb7a611 namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: ff5b74d5-e21b-4c0a-aae1-aba4443dbe36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T11:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: fc1c25f1-4609-49fd-9424-2fa78168573f kind: Event lastTimestamp: "2026-07-29T11:00:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T11:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:50Z" name: error-404-isvc-afa2d-predictor.18c6bda56a13b941 namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 824e3836-7dd4-4ca7-aab9-ee8436aacc9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T11:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: fc1c25f1-4609-49fd-9424-2fa78168573f kind: Event lastTimestamp: "2026-07-29T11:00:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T11:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:50Z" name: error-404-isvc-afa2d-predictor.18c6bda56a848aec namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: 64bbb5fe-56bf-45db-876a-0702e3280240 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 32588410-a33e-406f-ac78-2633187a8489 kind: Event lastTimestamp: "2026-07-29T11:00:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:24Z" name: error-404-isvc-afa2d.18c6bd9aede772ff namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: d8d60c5d-6320-4e74-af89-b7eba399935c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 32588410-a33e-406f-ac78-2633187a8489 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-afa2d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-afa2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d.18c6bd9aee5de258 namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: 42b89fa7-671c-484f-84fe-6278f4269601 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 32588410-a33e-406f-ac78-2633187a8489 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-afa2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-afa2d.18c6bd9aee5e36d6 namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: ef27666f-0b99-4baa-9444-0bb9a0eea41c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: 32588410-a33e-406f-ac78-2633187a8489 kind: Event lastTimestamp: "2026-07-29T11:00:23Z" message: InferenceService [error-404-isvc-afa2d] is Ready metadata: creationTimestamp: "2026-07-29T11:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:23Z" name: error-404-isvc-afa2d.18c6bda63d27d00e namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: bf9bf90b-c9de-44f6-a813-ea3670a1c84f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:48:00.023477Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf905762192 namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 06071d4a-11a3-430d-91f4-25e7a4a84153 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf91fe96523 namespace: kserve-ci-e2e-test resourceVersion: "19840" uid: 012d110b-f246-45d4-9650-9f7fc671f032 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf9214ba083 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 734bc7ff-ab34-412d-947e-cc252b4f3357 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf92445752c namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a0304930-247c-4572-b384-0ea099f5ffe4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf924f39afa namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: 2aff2988-cf95-4798-9f4c-cbe9d7fb5452 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf925003523 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: ad9594d6-f64c-47ee-a18d-e9b7c7513248 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf928b1f7fe namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: e24dd920-d381-4b8a-9ea8-6a663d338899 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf92930a8d4 namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: 3d890080-ac3c-4d6b-9925-2adca0b1fb51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:48:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:38Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bcf995f19e82 namespace: kserve-ci-e2e-test resourceVersion: "20153" uid: 31357d88-23b2-42a3-b20d-bc55866cd427 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bd095b7b70c9 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: beb4d082-be71-411d-8e7b-6aebb25aa11f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bd095b7c0d90 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 14469e4b-b631-4d51-87a2-9949f5eaa41b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 71307829-fbab-456d-ba06-bccbc23e9076 kind: Event lastTimestamp: "2026-07-29T10:49:13Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:13Z" name: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6.18c6bd0a1e30b4c5 namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 8121d663-3ee6-4c52-bc1a-dd94a2ef458f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b12e8-predictor-665bb476f9 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: a3d38cc5-3467-47b8-b5da-eada6bc37762 kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: 'Created pod: error-404-isvc-b12e8-predictor-665bb476f9-m2jk6' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8-predictor-665bb476f9.18c6bcf90552e672 namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 59dd4544-919c-462d-aefa-b0ffcd5a0e6e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19785" uid: 1e13470c-4b27-4bbf-b200-f7c9cb273022 kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: Scaled up replica set error-404-isvc-b12e8-predictor-665bb476f9 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:47:59Z" name: error-404-isvc-b12e8-predictor.18c6bcf90386838f namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: 3ef49fc0-2e6b-49ef-ad84-205abac7052b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 9bfdba9b-9d4a-4b76-9076-fae73e65f8c7 kind: Event lastTimestamp: "2026-07-29T10:48:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:30Z" name: error-404-isvc-b12e8-predictor.18c6bcfc823d28e0 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: e729386d-8fd5-4947-9d97-318255013e37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 9bfdba9b-9d4a-4b76-9076-fae73e65f8c7 kind: Event lastTimestamp: "2026-07-29T10:48:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:30Z" name: error-404-isvc-b12e8-predictor.18c6bcfc82add696 namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: c6059b20-96e5-49fc-937a-214525edd55d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:48:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 9bfdba9b-9d4a-4b76-9076-fae73e65f8c7 kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-b12e8-predictor.18c6bd03800ab856 namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 752c1e2b-d56d-4480-a966-6debf67265f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:48:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 9bfdba9b-9d4a-4b76-9076-fae73e65f8c7 kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:00Z" name: error-404-isvc-b12e8-predictor.18c6bd03807299d0 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 4ca23c23-b3d1-4b8f-86aa-5752bb3058f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: 04fcc44a-428d-436a-9609-92c995116328 kind: Event lastTimestamp: "2026-07-29T10:48:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:48Z" name: error-404-isvc-b12e8.18c6bcf903a1ce53 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 5b0d4ee2-6164-4e65-8997-7fb1a31014a3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: 04fcc44a-428d-436a-9609-92c995116328 kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-b12e8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b12e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8.18c6bcf903fc0f7c namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: e0950625-c7c3-4227-b5b6-cddb7f926d10 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: 04fcc44a-428d-436a-9609-92c995116328 kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b12e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:00Z" name: error-404-isvc-b12e8.18c6bcf903fc5bdf namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: b01aceca-829f-4ddf-a6c3-abd66d85091b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: 04fcc44a-428d-436a-9609-92c995116328 kind: Event lastTimestamp: "2026-07-29T10:48:48Z" message: InferenceService [error-404-isvc-b12e8] is Ready metadata: creationTimestamp: "2026-07-29T10:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:48Z" name: error-404-isvc-b12e8.18c6bd0450b4997c namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: f290b875-95e5-442c-b64a-37a83767c445 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:49:10.357321Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d2d23-predictor-658b5fdc7-kx72c to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd0965af67d8 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 457ad958-ddb1-45c3-be43-155c31b4273a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd097ffc7c21 namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: e3ba2789-f6c0-415f-b283-46019fc26399 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd09815a35b6 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: f1042f4c-e1d3-405c-8086-837ebe4caae9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd0984305e2c namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: 08d7d397-dac0-49c3-8579-4c1383e677c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd0984a504b0 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: e2f9e0f0-932a-4573-9d1b-69cbdae7718a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd0984afea34 namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: 5e4d78bd-df59-49d6-b668-4513d4a8d224 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd0987c37a00 namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: ac9d2267-58d4-45ae-b3c4-57b21e1ff721 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd0988363cd0 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: a5e1ace9-ff6b-462f-9e2e-82500320b43f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:49:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:48Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd09f816f6f7 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 1d97945e-4080-4f92-86f0-16db0ab3c962 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd8a96ec912f namespace: kserve-ci-e2e-test resourceVersion: "25062" uid: 23b6afc0-50ce-4491-a38e-8e142e911102 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 53028cc2-acf8-4064-81d6-76c1a003da51 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c.18c6bd8a96ed4687 namespace: kserve-ci-e2e-test resourceVersion: "25063" uid: 44c66088-84a6-4c8d-b059-7656358bc538 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d2d23-predictor-658b5fdc7 namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: d0eeb566-1e95-457a-8ab3-0a0ce9d38e31 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'Created pod: error-404-isvc-d2d23-predictor-658b5fdc7-kx72c' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor-658b5fdc7.18c6bd0965022eab namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 7ac72201-71d6-46fe-bda6-6ffa7755b7f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: 03479148-10b2-4a4c-88ef-67f5973c3eea kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Scaled up replica set error-404-isvc-d2d23-predictor-658b5fdc7 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23-predictor.18c6bd096338c4dc namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 5a6f63e1-d657-4265-8832-f47e72e73a45 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: a4a7b3c6-a292-4337-8af1-7a84d3c16bfb kind: Event lastTimestamp: "2026-07-29T10:49:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:40Z" name: error-404-isvc-d2d23-predictor.18c6bd0ce1b06873 namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: a27f8e1a-2f74-40c5-bc78-33aa60dff4ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: a4a7b3c6-a292-4337-8af1-7a84d3c16bfb kind: Event lastTimestamp: "2026-07-29T10:49:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:40Z" name: error-404-isvc-d2d23-predictor.18c6bd0ce21fb093 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: f8b9caaa-3c1a-46e4-984c-e3d94a4719f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: a4a7b3c6-a292-4337-8af1-7a84d3c16bfb kind: Event lastTimestamp: "2026-07-29T10:50:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:25Z" name: error-404-isvc-d2d23-predictor.18c6bd13df92f2eb namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 6ee605f7-3004-4a24-b306-5a813b7d4d24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: a4a7b3c6-a292-4337-8af1-7a84d3c16bfb kind: Event lastTimestamp: "2026-07-29T10:50:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:25Z" name: error-404-isvc-d2d23-predictor.18c6bd13dffbb4ee namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: e0a79ca3-4b81-4c75-a4ab-d47f273a26d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 293406aa-c967-4273-a9c6-f2b97d5f9b14 kind: Event lastTimestamp: "2026-07-29T10:49:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:58Z" name: error-404-isvc-d2d23.18c6bd096338b2af namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: 85774d2a-54fe-4332-9783-a8890f3fbd94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 293406aa-c967-4273-a9c6-f2b97d5f9b14 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-d2d23": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d2d23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23.18c6bd0963b8094b namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 4fd516dc-352d-4bcd-bbb3-ece6ea8930bb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 293406aa-c967-4273-a9c6-f2b97d5f9b14 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d2d23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:10Z" name: error-404-isvc-d2d23.18c6bd0963b86c7d namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 037876fd-dd44-427b-9fb1-4bdc65325fa8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 293406aa-c967-4273-a9c6-f2b97d5f9b14 kind: Event lastTimestamp: "2026-07-29T10:49:58Z" message: InferenceService [error-404-isvc-d2d23] is Ready metadata: creationTimestamp: "2026-07-29T10:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:58Z" name: error-404-isvc-d2d23.18c6bd14b34a5f7d namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: b85d31d6-97ef-4f77-9015-a5f07ebd46ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:37:15.027012Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f0182-predictor-6ff7979689-6jg5k to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:37:15Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc62d8ad1b1a namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: 142e14f0-a4e0-4895-8691-c4a7b00e02bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f0182-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc62e1466bc0 namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 2c4e8acd-a3cd-4a12-aa6c-20db54953fca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:37:16Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc631696765f namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: a7ac1e64-3366-42d1-9f8f-a0d097369089 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:16Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc6317f49a39 namespace: kserve-ci-e2e-test resourceVersion: "14464" uid: 94b4ac7f-7a3b-4b03-a149-433887a3e1c6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:27Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" in 11.573s (11.573s including waiting). Image size: 1333943915 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:27Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc65c9cbb1cc namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 46286a37-e0fe-4be2-a98c-23d00f457200 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:27Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc65cd937b80 namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: c7b0ce84-9e20-496c-a588-59bfe123a1d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:27Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc65ce1e1dbd namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: bbe9c515-389c-4271-adde-9fa2713fb7c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-29T10:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:27Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc65ce2d9675 namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 29b5f87d-5698-44d9-b462-bf783be15818 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.577s (2.577s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:30Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc6667d5ad5d namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 848bb15c-446a-46bb-9292-dcfc23dbab9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:30Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc666b7606bf namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: c8d90482-3189-42d3-84bd-73b796a5ca71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:37:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:30Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc666c068ba9 namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 52337a43-9625-494c-b06c-b74173ece55f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:37:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:38:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:37:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:08Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc66e131bb06 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: d2d5db88-247b-4993-9152-a1f8848e6442 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:44Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc77c90a3ae9 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 98e7c30d-cbc5-4449-9019-0e23d2a0f3e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:44Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc77c90ad389 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 88c48d00-bee8-4bbd-9a65-6217bb964da7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:38:49Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-29T10:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:49Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc78cf047487 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: eb7f11dc-76b4-48c3-a13f-f0d697e127b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: dec3d21b-ccca-41e5-8be9-56c970cf4d5a kind: Event lastTimestamp: "2026-07-29T10:38:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: i/o timeout' metadata: creationTimestamp: "2026-07-29T10:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:49Z" name: error-404-isvc-f0182-predictor-6ff7979689-6jg5k.18c6bc78cf6cc849 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 6da0d781-2690-4dee-8ee7-213e9e60b262 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f0182-predictor-6ff7979689 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 526f5636-45aa-4a34-9ea3-9441f88fc836 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: 'Created pod: error-404-isvc-f0182-predictor-6ff7979689-6jg5k' metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:15Z" name: error-404-isvc-f0182-predictor-6ff7979689.18c6bc62d84631f6 namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: e072cd92-db86-4943-9ae9-c577dbb934ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: 80e16461-098f-4053-80fa-032caef58075 kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: Scaled up replica set error-404-isvc-f0182-predictor-6ff7979689 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: error-404-isvc-f0182-predictor.18c6bc62d6311f21 namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 2fac8d3b-a3c9-45e2-b572-884853f15b37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: cde0db91-176f-4d6e-a7ca-262e3baec2e4 kind: Event lastTimestamp: "2026-07-29T10:38:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:00Z" name: error-404-isvc-f0182-predictor.18c6bc6654b69618 namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 179735a2-2067-432b-9ed3-e43b8273b494 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:37:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: cde0db91-176f-4d6e-a7ca-262e3baec2e4 kind: Event lastTimestamp: "2026-07-29T10:38:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:00Z" name: error-404-isvc-f0182-predictor.18c6bc665526b561 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 882d21e4-d937-41fb-91df-5955a4fe8fcc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: cde0db91-176f-4d6e-a7ca-262e3baec2e4 kind: Event lastTimestamp: "2026-07-29T10:38:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:30Z" name: error-404-isvc-f0182-predictor.18c6bc69d3d412fa namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 8b022392-f0ca-4cf7-9ad5-491c3bbf4364 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: cde0db91-176f-4d6e-a7ca-262e3baec2e4 kind: Event lastTimestamp: "2026-07-29T10:38:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:30Z" name: error-404-isvc-f0182-predictor.18c6bc69d447eeed namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 2beeb3f1-c35a-4539-8e99-ee1fa5d803b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0182 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 827c28b5-ef78-4967-80ca-c685cba426cd kind: Event lastTimestamp: "2026-07-29T10:38:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:18Z" name: error-404-isvc-f0182.18c6bc62d631bcd2 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: deb9b651-ff4a-494e-89ef-c8037e481cf6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0182 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 827c28b5-ef78-4967-80ca-c685cba426cd kind: Event lastTimestamp: "2026-07-29T10:38:18Z" message: InferenceService [error-404-isvc-f0182] is Ready metadata: creationTimestamp: "2026-07-29T10:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:18Z" name: error-404-isvc-f0182.18c6bc719ccdf71f namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 30c53520-f012-436a-9994-4e533d8b4de6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:39:44.988980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:39:44Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85c31b293b namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 76823119-987c-413c-9bb0-fbf26d196c04 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85dd759445 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 61d6e35a-7e3c-4f74-80d4-594854b990af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85decd4aa6 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 0975d097-df93-430e-8c73-22454fcc53ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85e1a55553 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 9d50ca44-34a4-4aa0-8c01-5472cebac63c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85e25efb70 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: cfedc289-e42c-4bc4-80e1-5fa6519a0cf4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85e26acdf3 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: 5947910d-8a70-4660-b8c7-2680fb00b583 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85e5550bf3 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: cfe52157-8229-46e3-b839-3138d98ff5b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc85e5da427e namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: c87ee724-e412-47d6-ae17-534adf295d5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:49:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:03Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bc863bb51059 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: cc998080-57f2-49bc-87db-2b5506f07152 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bd06e9208ae1 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 1d8ea904-0954-43ab-b2d0-f9f4cb81824b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:59Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bd06e9213635 namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: a9c8d51d-62ab-4a88-a656-064e3cfce3e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9c7e36d2-0624-43cc-acbf-3c4117c8a4e7 kind: Event lastTimestamp: "2026-07-29T10:49:03Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:03Z" name: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj.18c6bd07b01e9c46 namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: ebbeaff3-cca5-467f-92de-56d4e9414356 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f1d40-predictor-749bd5f9df namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 579a0415-b45f-4dca-8f2b-d9bed96c9692 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: 'Created pod: error-404-isvc-f1d40-predictor-749bd5f9df-wqmpj' metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:44Z" name: error-404-isvc-f1d40-predictor-749bd5f9df.18c6bc85c2ab348e namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 1da8f395-7e32-4b2d-9cc3-8fe0472def18 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: bba1fe1f-77a4-41a5-a6ab-60c68825d81a kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Scaled up replica set error-404-isvc-f1d40-predictor-749bd5f9df from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:44Z" name: error-404-isvc-f1d40-predictor.18c6bc85c10c8263 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 02a5a191-8b26-4a50-b922-eec55536c92a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: e2f8b8a1-60c1-4781-908e-dcac5c773a9f kind: Event lastTimestamp: "2026-07-29T10:40:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:14Z" name: error-404-isvc-f1d40-predictor.18c6bc893f8bca46 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 56456b29-a85e-4e64-b3f6-6ce0f9fcceb4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: e2f8b8a1-60c1-4781-908e-dcac5c773a9f kind: Event lastTimestamp: "2026-07-29T10:40:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:14Z" name: error-404-isvc-f1d40-predictor.18c6bc893ff11a27 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: 0a1680d3-ca47-4871-a3d0-d9b722b2b4fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:40:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: e2f8b8a1-60c1-4781-908e-dcac5c773a9f kind: Event lastTimestamp: "2026-07-29T10:40:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:45Z" name: error-404-isvc-f1d40-predictor.18c6bc903d825353 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: fa013b4e-d214-4f09-946c-39cdac9f95c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:40:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: e2f8b8a1-60c1-4781-908e-dcac5c773a9f kind: Event lastTimestamp: "2026-07-29T10:40:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:45Z" name: error-404-isvc-f1d40-predictor.18c6bc903dded163 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 419870ab-f8fb-468c-ba17-e22ebfd39582 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 068ca935-fdda-4c4d-bdd9-36138fcf5b4e kind: Event lastTimestamp: "2026-07-29T10:40:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:40:33Z" name: error-404-isvc-f1d40.18c6bc85c10da7d8 namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 44597e2e-0b3b-458c-b4d3-234a0cab637b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 068ca935-fdda-4c4d-bdd9-36138fcf5b4e kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-f1d40": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f1d40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:44Z" name: error-404-isvc-f1d40.18c6bc85c200498d namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 04e090cf-04f4-47a7-9c06-c6d529ed98ee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 068ca935-fdda-4c4d-bdd9-36138fcf5b4e kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f1d40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:44Z" name: error-404-isvc-f1d40.18c6bc85c2009be2 namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: 2d55d6fc-326b-4d00-b53b-d10324cd99f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 068ca935-fdda-4c4d-bdd9-36138fcf5b4e kind: Event lastTimestamp: "2026-07-29T10:40:33Z" message: InferenceService [error-404-isvc-f1d40] is Ready metadata: creationTimestamp: "2026-07-29T10:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:40:33Z" name: error-404-isvc-f1d40.18c6bc90f6c4c75d namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 6217789b-216e-42aa-b244-b1b2592a338e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:58:25.194648Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f5066-predictor-645f7d7b85-fsdjq to ip-10-0-143-36.ec2.internal metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8a949125f2 namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 14f91ca3-16ad-4285-9eec-f126625e7719 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25060" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8aaebb9cfb namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: 693dd9aa-9e9a-451a-95d0-95bbef20424c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8ab0cc315c namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: a846689f-924c-4f5d-b16d-9518b5c4a4f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8ab51df95e namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: 61f1ef1c-824a-414e-9716-e8b0fe5d857b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8ab595fc88 namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: b0a1d5e8-1c3a-4f49-ae79-9303df56be80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8ab5a04aaa namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: b6766dad-2119-41d5-a569-1fdd5e0ecb05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8ab8cb520b namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: 104423c4-0b49-4aae-910e-3e22134f6fb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8ab94ad27f namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: 612253ee-8b9e-48d9-bcce-cdafcdf3df2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:58:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:59:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:03Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd8b1f0ca559 namespace: kserve-ci-e2e-test resourceVersion: "25396" uid: 4ad7df80-9949-4b9b-bedb-9ea0ac9d539a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd9ae58a5461 namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: 08d4c16d-7af9-4ae6-bda0-7ff0ddd466cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd9ae58b06bc namespace: kserve-ci-e2e-test resourceVersion: "25876" uid: 698154f3-1647-4a42-8ad8-e15d6b6d12de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 5b77336e-7fa8-40fa-8886-cbf3b2b9b729 kind: Event lastTimestamp: "2026-07-29T10:59:38Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:38Z" name: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq.18c6bd9ba797f8dc namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 0c769ca8-99ee-405c-a5bb-9548884eb113 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-36.ec2.internal source: component: kubelet host: ip-10-0-143-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f5066-predictor-645f7d7b85 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: f681bc8d-9c99-4ff8-848c-e67c3a78715f kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: 'Created pod: error-404-isvc-f5066-predictor-645f7d7b85-fsdjq' metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor-645f7d7b85.18c6bd8a942a9d8e namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 79031d5c-926d-4bda-b785-f1f385186491 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: ee972d23-e4d9-4008-af96-43dcbc0a9820 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Scaled up replica set error-404-isvc-f5066-predictor-645f7d7b85 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:25Z" name: error-404-isvc-f5066-predictor.18c6bd8a927aff57 namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: ada9edfb-1170-407c-a861-05930c8d1007 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:58:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: 958b5387-2d3d-4c3d-9f6b-5ad906d757f2 kind: Event lastTimestamp: "2026-07-29T10:58:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:55Z" name: error-404-isvc-f5066-predictor.18c6bd8e11009dfd namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 226de086-c2ce-4bab-aa68-6540f8fde101 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:58:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: 958b5387-2d3d-4c3d-9f6b-5ad906d757f2 kind: Event lastTimestamp: "2026-07-29T10:58:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:55Z" name: error-404-isvc-f5066-predictor.18c6bd8e11679407 namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 10403b1d-f185-41ba-8628-9996e6011f7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: 958b5387-2d3d-4c3d-9f6b-5ad906d757f2 kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-f5066-predictor.18c6bd950ed74636 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: beb3aafd-a422-4ee0-8379-3fbfea8a9c13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: 958b5387-2d3d-4c3d-9f6b-5ad906d757f2 kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: error-404-isvc-f5066-predictor.18c6bd950f409575 namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: 37d6ca3c-202f-417d-9927-bc3605f06b65 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 10c3f925-a78f-48b1-a1fa-77665bc408f3 kind: Event lastTimestamp: "2026-07-29T10:59:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:13Z" name: error-404-isvc-f5066.18c6bd8a927cb9f7 namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 9056e82d-9cde-4850-8c4f-3a5da22b5c7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: 10c3f925-a78f-48b1-a1fa-77665bc408f3 kind: Event lastTimestamp: "2026-07-29T10:59:13Z" message: InferenceService [error-404-isvc-f5066] is Ready metadata: creationTimestamp: "2026-07-29T10:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:13Z" name: error-404-isvc-f5066.18c6bd95da639157 namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: 4c606adc-6840-4d4b-b942-10ee4c34e974 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:37:14.667037Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 to ip-10-0-137-188.ec2.internal metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:37:14Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc62c3385a62 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: ca85861d-f766-413c-8945-dcc156d42a3f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:14Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc62cc7f9b6d namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: a11af080-c337-46c6-8beb-985d8d3a44f7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6300588e9d namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 72f59432-fade-4e73-8d7e-a857f9fb408c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13da3ee375cce7c9d4590e64ebb24ecf2e97ed4b2b64f9a4f2e3074ed4aafac1" already present on machine metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6301beffbc namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: d8039a44-a558-4100-9623-5673a972af59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6304d205b0 namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 17341e86-159c-4430-91f5-8a0d4da15b21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc630543d27d namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 9770d594-ea46-4346-88f1-12fac43a012f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:20Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1828" metadata: creationTimestamp: "2026-07-29T10:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:20Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6421f26234 namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: ee7e741b-36c1-49ab-bf11-363f10782458 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:44Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1828" in 24.127s (24.127s including waiting). Image size: 1555733430 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:44Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc69c011d938 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 2b402630-312b-4aaf-8bac-50b762913e5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:44Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc69c4e7732b namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 782854cb-88a1-4a43-b5b5-694701cdb19f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:44Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc69c6475d80 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: b5cc6cbe-c2e5-4c56-8478-b51bb2108d61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-29T10:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:44Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc69c6569f5d namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 99907c86-ee64-4327-b337-0be1d4231b52 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.089s (3.089s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:47Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6a7e8067ff namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 468a8499-9499-420f-864c-d0f2943dbeea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:48Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6a8b6ccd9f namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 8411b30c-3d85-4ae1-b8fc-df6235bdd78a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:37:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:48Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6a8cbfaa04 namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: 411bf761-b7cd-4b93-9c87-63bb449cc331 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-29T10:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc6aed027621 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 98953cae-a2d7-4442-b274-0324d794bb64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc85be2682d3 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: e7f6403a-c750-4ede-9246-0333b8db169d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc85be271036 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: b5fff0fe-6374-42fd-a5f3-5a51087b24d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0c4f1a56-ad37-4199-8770-34038a003753 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6.18c6bc85ef4c784b namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: bd57edb3-d027-40ba-8230-3e5013d225ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-65bbfc86dd namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 62920db6-0aa0-49a3-844e-b51879a83b12 kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-65bbfc86dd-2qcj6' metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: isvc-sklearn-graph-1-predictor-65bbfc86dd.18c6bc62c2c65076 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: e2460fb5-fe39-4be8-90e0-8f6be55ce349 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 11525af4-a54c-4639-9aad-b4ea5a89a12b kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-65bbfc86dd from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: isvc-sklearn-graph-1-predictor.18c6bc62be574e48 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: bdcb3d95-f5d5-4278-9254-35ab58ad568e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 011036a4-3571-46df-a6d7-56edb3f43feb kind: Event lastTimestamp: "2026-07-29T10:38:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:14Z" name: isvc-sklearn-graph-1-predictor.18c6bc6643787e21 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: e9705465-ff9c-4863-8fdf-1d3307831f92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 011036a4-3571-46df-a6d7-56edb3f43feb kind: Event lastTimestamp: "2026-07-29T10:38:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:14Z" name: isvc-sklearn-graph-1-predictor.18c6bc6643f1a223 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 82370153-e342-41aa-b292-154c9a1314f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:37:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: 011036a4-3571-46df-a6d7-56edb3f43feb kind: Event lastTimestamp: "2026-07-29T10:39:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:29Z" name: isvc-sklearn-graph-1-predictor.18c6bc6d41ceece4 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: cfb4090e-f84f-4112-bc6c-0b5c49437392 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:37:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: 011036a4-3571-46df-a6d7-56edb3f43feb kind: Event lastTimestamp: "2026-07-29T10:39:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:29Z" name: isvc-sklearn-graph-1-predictor.18c6bc6d42206c97 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ff09606f-b239-4511-b228-f64924257ab2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 011036a4-3571-46df-a6d7-56edb3f43feb kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: deployments.apps "isvc-sklearn-graph-1-predictor" not found metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:44Z" name: isvc-sklearn-graph-1-predictor.18c6bc85bbfbe484 namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: dc12e17b-2537-4a49-b9cb-75d7ab058aaf reason: FailedGetScale reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 011036a4-3571-46df-a6d7-56edb3f43feb kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: horizontalpodautoscalers.autoscaling "isvc-sklearn-graph-1-predictor" not found metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:44Z" name: isvc-sklearn-graph-1-predictor.18c6bc85bc73e5e5 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: e48f802b-4f45-4504-b64f-47e03a53b63d reason: FailedUpdateStatus reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: c944cd80-5987-404d-9922-2d32ca697dbb kind: Event lastTimestamp: "2026-07-29T10:39:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:06Z" name: isvc-sklearn-graph-1.18c6bc62c411c7af namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 181c88ac-4e60-4397-8dec-8cac5d943954 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: c944cd80-5987-404d-9922-2d32ca697dbb kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:37:14Z" name: isvc-sklearn-graph-1.18c6bc62c49807f9 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 6e40a157-3e7b-4039-8e60-93254bc4e889 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: c944cd80-5987-404d-9922-2d32ca697dbb kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:37:14Z" name: isvc-sklearn-graph-1.18c6bc62c498616a namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 438de14d-b1db-4838-9183-1315fe7e6a39 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: c944cd80-5987-404d-9922-2d32ca697dbb kind: Event lastTimestamp: "2026-07-29T10:39:06Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-29T10:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:06Z" name: isvc-sklearn-graph-1.18c6bc7ccfc772bf namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 655d18f1-6ce4-45a9-be26-159917750f70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:37:15.137856Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc62df487a9a namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: ad552030-c8a2-4dc2-b165-aa07d34b722b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc62ea956478 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 4626a0e4-9eb1-4f24-ad3d-6b6dca6a96bd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:37:16Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc6321d425d8 namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 66424d54-d6b8-49dd-b7c8-2156c024fad0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:13da3ee375cce7c9d4590e64ebb24ecf2e97ed4b2b64f9a4f2e3074ed4aafac1" metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:16Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc6323f5b400 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: c8efef70-6898-4bed-ba0c-7b05f23f482b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:28Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:13da3ee375cce7c9d4590e64ebb24ecf2e97ed4b2b64f9a4f2e3074ed4aafac1" in 12.685s (12.685s including waiting). Image size: 300978649 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:28Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc661815c9bd namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 9ec507c9-b8b1-4e2b-903a-5c4ee406fe6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:29Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc661ae6158a namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: bb0e96a7-97ec-4dfc-aa26-f673ccf0fdad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:29Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc661b6533df namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 78bd0f9d-a471-400a-ac9b-58c49dee3a73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:33Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1828" metadata: creationTimestamp: "2026-07-29T10:37:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:33Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc671e05ab3e namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 5074c330-b615-452e-a549-03c26652e351 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:39Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1828" in 6.29s (6.29s including waiting). Image size: 1555733430 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:39Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc6894f90485 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: e31aaf20-3f15-4a7e-adf9-addd3dca50d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:39Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc6898a73be5 namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 4a23e1c6-7f30-41a8-bcc7-73e1b544e779 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:39Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc6899205267 namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 021cf80a-532f-4108-9916-2f3b3920bded reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:39Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc68992cfc23 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 2901d889-f1d3-4198-8f3d-9b8288b4cadb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:39Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc689c66260f namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 87a2de74-87ea-42f0-a3a1-fc3569f6a53a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:37:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:39Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc689ce640f3 namespace: kserve-ci-e2e-test resourceVersion: "14719" uid: 98fae084-8d17-4caa-a97f-6c77d5c1770a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-29T10:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:38:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:51Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc68fcf3b0eb namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: cf52dd3c-3817-461c-b910-4d962727a807 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc85b9d26353 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 42ac5103-027a-4a74-acc3-6884a79a1726 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc85b9d300d8 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 4dba7dbc-5ea4-4357-841b-799df4bc3d18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 00c8b1c9-1c37-41f0-a7cf-c8e993ed1808 kind: Event lastTimestamp: "2026-07-29T10:39:47Z" message: 'Readiness probe failed: Get "https://10.134.0.17:8643/healthz": dial tcp 10.134.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:47Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52.18c6bc865349df76 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 76232ca4-188e-47fc-b0e8-7249020772f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-77b6fbc468 namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 991da105-c0d1-4525-964e-0f80c5f17927 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-77b6fbc468-69w52' metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-2-predictor-77b6fbc468.18c6bc62dee015cb namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: cc0f8a62-1e75-430a-8dcb-2a8c98f4f8d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: ffea04ff-faa3-4af5-b952-c7b8870c2229 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-77b6fbc468 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:15Z" name: isvc-sklearn-graph-2-predictor.18c6bc62dc0f1200 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 126e7f57-603f-46c4-80ee-5b51e64fa6bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:37:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 113d753c-f9d8-4bd8-93a6-fb8f8b258c14 kind: Event lastTimestamp: "2026-07-29T10:38:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:00Z" name: isvc-sklearn-graph-2-predictor.18c6bc665ad11e85 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: c63422c7-6e81-4344-aff9-edff1471cc1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:37:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 113d753c-f9d8-4bd8-93a6-fb8f8b258c14 kind: Event lastTimestamp: "2026-07-29T10:38:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:00Z" name: isvc-sklearn-graph-2-predictor.18c6bc665b471f3b namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 825568eb-41b9-4234-87c7-d44d9e416313 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:38:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 113d753c-f9d8-4bd8-93a6-fb8f8b258c14 kind: Event lastTimestamp: "2026-07-29T10:39:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:30Z" name: isvc-sklearn-graph-2-predictor.18c6bc70d76693e2 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 50efab11-485d-46cc-9776-83fa4540a076 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:38:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 113d753c-f9d8-4bd8-93a6-fb8f8b258c14 kind: Event lastTimestamp: "2026-07-29T10:39:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:30Z" name: isvc-sklearn-graph-2-predictor.18c6bc70d7cd034f namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: bb2206c5-6dc5-44ea-a4ec-a9b918c52590 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 94280a93-a192-4417-86f2-310ddf93d46e kind: Event lastTimestamp: "2026-07-29T10:39:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:01Z" name: isvc-sklearn-graph-2.18c6bc62dc0d7337 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: edf78cd3-7c2b-4bf0-b19b-b52811e9b1ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 94280a93-a192-4417-86f2-310ddf93d46e kind: Event lastTimestamp: "2026-07-29T10:39:01Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-29T10:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:01Z" name: isvc-sklearn-graph-2.18c6bc7b9c581fbf namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 652b09b0-4e63-4a7a-b057-2f9761c3ba91 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:37:14.946642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-rqkpf to ip-10-0-137-188.ec2.internal metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:37:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc62d3e2e731 namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: b84b900e-a98a-4244-81d0-0298e2e447a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc62de885fab namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: 8b87c303-bbc2-42fe-93af-d956be6a7b01 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:37:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc63466dcbc4 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: 184c4ba3-615b-4d93-bcfd-b8e22f5e80c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13da3ee375cce7c9d4590e64ebb24ecf2e97ed4b2b64f9a4f2e3074ed4aafac1" already present on machine metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc6347e415e5 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: a894dc6b-5e55-4a22-b6d7-4246d7e00c2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc634aacd1ec namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 7b9935f8-3725-4b9b-8d14-43eda219bf92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-29T10:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc634b17cb96 namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 8f2ee9b5-3948-4e52-9816-74b25955dc9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:21Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-29T10:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc645e42dd7e namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: 539f35f3-ccf4-4827-b233-f02b986ceb9f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:45Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.456s (23.456s including waiting). Image size: 1307355899 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc69d4617cc2 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: e735dcf9-2956-49f8-becd-98426fa95188 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc69dada03a2 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 18e4fcef-3d03-4869-b6fc-6526aa8bbdeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc69dbafe335 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 0ff1365d-36bf-4402-93fc-380cd44a131d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-29T10:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc69dbbd5c69 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 392d71b0-1609-41c4-aeaf-0ac3a3bfb7bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.868s (2.868s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc6a86b84290 namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 75ac8ece-df4e-4f4e-8d37-267306cdb663 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc6a8c63a5be namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: f4c02f59-9bfc-4101-bc31-c2e25ad88ddf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:37:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc6a8e094308 namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: f73f59f4-247b-4f15-9a60-8027e6dad524 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-29T10:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc6ab10fc751 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 16a69317-d1f2-43e5-b97e-c5bcb08702a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc85b7635a6a namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 9dd1d507-81f9-463d-8f89-b7cd68535bbb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: d78a258f-b209-4002-94d0-57b4ad8a3ad7 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-rqkpf.18c6bc85b763e375 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 2599a11e-f44e-4f7c-bedf-d3f60788a826 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-188.ec2.internal source: component: kubelet host: ip-10-0-137-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: c3a33b41-bb9c-4264-b5ab-f8b07486297b kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-rqkpf' metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c6bc62d34a68f4 namespace: kserve-ci-e2e-test resourceVersion: "14366" uid: dc210cea-ac28-4646-b814-40a737058d32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 6c75bc9c-6335-46b3-a118-311ca1f002b5 kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: isvc-xgboost-graph-predictor.18c6bc62d09e5a9c namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 7b9b48a9-ea8f-4613-9b1d-8c9e24813052 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: 733e1983-0c96-4f11-ad2f-d21aa1ac9a3b kind: Event lastTimestamp: "2026-07-29T10:38:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:29Z" name: isvc-xgboost-graph-predictor.18c6bc664ef99c81 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 9a706044-5713-4d45-b067-04677b264216 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: 733e1983-0c96-4f11-ad2f-d21aa1ac9a3b kind: Event lastTimestamp: "2026-07-29T10:38:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:29Z" name: isvc-xgboost-graph-predictor.18c6bc664f6e5777 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 47fffdeb-d449-4c13-9fd6-b0f3002df586 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 733e1983-0c96-4f11-ad2f-d21aa1ac9a3b kind: Event lastTimestamp: "2026-07-29T10:39:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:14Z" name: isvc-xgboost-graph-predictor.18c6bc77c90d12b7 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 785a4f3b-c80a-44bd-b415-678d7cd913f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 733e1983-0c96-4f11-ad2f-d21aa1ac9a3b kind: Event lastTimestamp: "2026-07-29T10:39:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:14Z" name: isvc-xgboost-graph-predictor.18c6bc77c96dd344 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 6a88b251-0b93-48c1-b9ab-5c4345c08c4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: ed40f56e-4623-4b98-9a64-192c5c285d31 kind: Event lastTimestamp: "2026-07-29T10:38:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:54Z" name: isvc-xgboost-graph.18c6bc62d05f6804 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: e367c163-6410-4c49-a2fe-3ce257dc5145 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: ed40f56e-4623-4b98-9a64-192c5c285d31 kind: Event lastTimestamp: "2026-07-29T10:38:54Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-29T10:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:54Z" name: isvc-xgboost-graph.18c6bc7a146e5669 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: f5a76f06-9d7a-42af-afd3-ec4bb5096d6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:39:34.612926Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-79dd7c6579-pzmm2 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:39:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:39:34Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc8358a5119c namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 0ba53e88-0a86-4ff1-b87a-1fc0fcbd5233 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:35Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: "2026-07-29T10:39:35Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:39:35Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc837252963a namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 9ce149bf-fb27-402d-9028-f07b5965335f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: "2026-07-29T10:39:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:35Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc837399ee31 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 6dcbe46b-b7d8-4d8e-be9e-621a2217bbf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: "2026-07-29T10:39:35Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-29T10:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:35Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc83767f7d8d namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 00f1469b-3b24-449a-996f-900bbefea1f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: "2026-07-29T10:39:35Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-29T10:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:35Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc8377054c63 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: f6dd39a0-1ef3-40da-b152-db0cfc942a24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:44Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc85b206e460 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 9e943d8e-c914-4351-88f0-580fe20a71a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79dd7c6579-pzmm2 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 335eb5f3-3f74-4c50-b93e-0eca3d60899e kind: Event lastTimestamp: "2026-07-29T10:40:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:40:11Z" name: model-chainer-79dd7c6579-pzmm2.18c6bc8631eefb62 namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: e7d7fb36-69f6-47a3-b501-a5b83c353f29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-79dd7c6579 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: d66ad91f-1b20-48f5-bcc3-5c985a32b588 kind: Event lastTimestamp: "2026-07-29T10:39:34Z" message: 'Created pod: model-chainer-79dd7c6579-pzmm2' metadata: creationTimestamp: "2026-07-29T10:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:34Z" name: model-chainer-79dd7c6579.18c6bc83580b0160 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 5b785d9f-c7e3-4ed5-bced-7e2913af79c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 7741ca0a-25a1-4bca-8cd6-b37c0c01b6b7 kind: Event lastTimestamp: "2026-07-29T10:39:34Z" message: Scaled up replica set model-chainer-79dd7c6579 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:34Z" name: model-chainer.18c6bc8357146504 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: ee598d76-e05e-4cbc-a1d1-50f972cc8be7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 3c4c1108-e1a0-44f4-b397-e9225c68e8fc kind: Event lastTimestamp: "2026-07-29T10:39:34Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-29T10:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:34Z" name: model-chainer.18c6bc835c821b9b namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: e891c865-1180-437b-942e-62c9a90f94af reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:51:10.133314Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0934b-74995788f-j8psd to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd2548e3f5a5 namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: 2e5607b4-196c-4dbb-ac7e-52bf6d14b788 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd256367be43 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: ab5da883-253f-4b4a-8cec-233db8f0bf39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0934b} kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd2564beaf44 namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 4cf2c86b-969f-4223-ab44-ef980f5103b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0934b} kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: 'Created container: sequence-graph-0934b' metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd256808d1b0 namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: 950a8ff9-2966-47f2-975f-c56ad28c4c8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0934b} kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: Started container sequence-graph-0934b metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd2568881081 namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: 8fffb06a-cecb-4a7e-a9df-6fa5ae8a1a99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0934b} kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Stopping container sequence-graph-0934b metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:24Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd9872d7e49f namespace: kserve-ci-e2e-test resourceVersion: "25583" uid: 7657abe9-82aa-4a03-a904-fc93096fb91d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0934b} kind: Pod name: sequence-graph-0934b-74995788f-j8psd namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: bc81178b-a296-4689-900c-7411ef6a3af8 kind: Event lastTimestamp: "2026-07-29T10:59:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:52Z" name: sequence-graph-0934b-74995788f-j8psd.18c6bd9917a8b115 namespace: kserve-ci-e2e-test resourceVersion: "26361" uid: 38b4f2bc-24e9-4eb5-882c-f3b540aa5056 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0934b-74995788f namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 2389cc34-1473-491f-9699-99ced3ca2f81 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: 'Created pod: sequence-graph-0934b-74995788f-j8psd' metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b-74995788f.18c6bd2548771b3a namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 1cc33e51-1ab5-4ff7-a4cf-e51499e3460c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0934b namespace: kserve-ci-e2e-test resourceVersion: "21966" uid: 3b254bca-8f90-4de8-bab9-aca61d529282 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: Scaled up replica set sequence-graph-0934b-74995788f from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b.18c6bd25475878a0 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 268dc387-5ffc-402e-8e25-fb428103e0ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0934b namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 04405fca-7438-465d-96a9-3a7a006281c4 kind: Event lastTimestamp: "2026-07-29T10:51:10Z" message: InferenceGraph [sequence-graph-0934b] is Ready metadata: creationTimestamp: "2026-07-29T10:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:51:10Z" name: sequence-graph-0934b.18c6bd254c7db162 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: ca2c3d29-0f0a-4e8a-ba03-265a0c056b96 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0934b namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 5b7353c9-262e-4b2e-833f-28843ee30263 kind: Event lastTimestamp: "2026-07-29T10:51:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:25Z" name: sequence-graph-0934b.18c6bd28c5d34975 namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: c55ecf8b-4100-4be0-8410-26e8f1e4b550 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0934b namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 5b7353c9-262e-4b2e-833f-28843ee30263 kind: Event lastTimestamp: "2026-07-29T10:51:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:25Z" name: sequence-graph-0934b.18c6bd28c6333c12 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: a3061b66-22f1-4841-a496-c555efe2805f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0934b namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 5b7353c9-262e-4b2e-833f-28843ee30263 kind: Event lastTimestamp: "2026-07-29T10:51:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:51:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:40Z" name: sequence-graph-0934b.18c6bd2c44eda5e2 namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: 2bf721ed-b2d8-4935-bd55-998fceaa0e79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:51:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0934b namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 5b7353c9-262e-4b2e-833f-28843ee30263 kind: Event lastTimestamp: "2026-07-29T10:51:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:51:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:40Z" name: sequence-graph-0934b.18c6bd2c4555c1eb namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: b956be47-508a-44b0-a407-3832f5e2f661 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:49:59.811902Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6242a-6df9fc859f-jpk45 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:49:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:49:59Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd14e96877c8 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 896538ed-4f5a-495d-8121-546e8c90f0de reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:00Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6242a-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:00Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd14f8a7f44f namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 21ef0020-aab4-47af-8edb-f87c733e340b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:00Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:00Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:50:00Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd152730b5e1 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: e621a988-e2ce-4b88-8a09-ffd38d2a4f9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6242a} kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:00Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd152897c6de namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 57407b82-5727-4d8b-be9d-362a4853f706 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6242a} kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:00Z" message: 'Created container: sequence-graph-6242a' metadata: creationTimestamp: "2026-07-29T10:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:00Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd152b90397f namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 5077ebb4-530a-45e7-83d6-91d3c5c3fefa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6242a} kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:00Z" message: Started container sequence-graph-6242a metadata: creationTimestamp: "2026-07-29T10:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:00Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd152c0d7159 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 82be9e52-e29e-4321-86a3-bdcac1035be9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6242a} kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:09Z" message: Stopping container sequence-graph-6242a metadata: creationTimestamp: "2026-07-29T10:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:09Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd1741f4d27a namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 23fdc56a-33e9-427f-8388-ffd331c41919 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6242a} kind: Pod name: sequence-graph-6242a-6df9fc859f-jpk45 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: c10b7ba0-85a3-4dd2-a8b6-3c9809084551 kind: Event lastTimestamp: "2026-07-29T10:50:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:37Z" name: sequence-graph-6242a-6df9fc859f-jpk45.18c6bd17cd6f5047 namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: 83c36366-b848-42ad-8561-a4704c7de38c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6242a-6df9fc859f namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 5d07fac5-b4dd-4a9b-b9dd-80d2adfac98e kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: 'Created pod: sequence-graph-6242a-6df9fc859f-jpk45' metadata: creationTimestamp: "2026-07-29T10:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:59Z" name: sequence-graph-6242a-6df9fc859f.18c6bd14e8fc2a1f namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 54e38461-1ddb-42ec-a506-47306533e5ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6242a namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 434a2691-2a10-4b1c-a315-ee46897def71 kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: Scaled up replica set sequence-graph-6242a-6df9fc859f from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:59Z" name: sequence-graph-6242a.18c6bd14e8175fb7 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: cabe0403-0f2f-4375-84db-24d53b66d93e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6242a namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: cee6c6c6-e315-45ed-b0a2-94f903642a4b kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: InferenceGraph [sequence-graph-6242a] is Ready metadata: creationTimestamp: "2026-07-29T10:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:59Z" name: sequence-graph-6242a.18c6bd14ed43d71f namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 8330351e-9d7d-46b6-ab6f-6e88470a125e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:40:44.892486Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f1d40-78bdff7fd9-5sdnw to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:40:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:40:44Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bc93b5a2184c namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 61c8d437-0248-45c0-8223-03bd38bcc896 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: "2026-07-29T10:40:45Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:40:45Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bc93cfaf80c1 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 84c272ef-5856-4730-bccc-0c5275597a04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f1d40} kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: "2026-07-29T10:40:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:40:45Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bc93d0fa482d namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 6e6625bf-d000-4403-ab49-3a7c75a33a62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f1d40} kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: "2026-07-29T10:40:45Z" message: 'Created container: sequence-graph-f1d40' metadata: creationTimestamp: "2026-07-29T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:40:45Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bc93d4520c6b namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 7feb75fc-7316-462a-a8f7-a3d50c005d01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f1d40} kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: "2026-07-29T10:40:45Z" message: Started container sequence-graph-f1d40 metadata: creationTimestamp: "2026-07-29T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:40:45Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bc93d4c685f2 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: a5b4ff26-4b43-4114-98bc-6f5c9ebe4cc9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f1d40} kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Stopping container sequence-graph-f1d40 metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:59Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bd06e39d2e14 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: 71df29de-d7cd-428f-a303-f6b574893746 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f1d40} kind: Pod name: sequence-graph-f1d40-78bdff7fd9-5sdnw namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 772851ff-213c-4341-9eeb-85bcbe08cf6c kind: Event lastTimestamp: "2026-07-29T10:49:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:27Z" name: sequence-graph-f1d40-78bdff7fd9-5sdnw.18c6bd077ad02a5b namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: f165bd13-5387-47a3-b177-a534e5ca995a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f1d40-78bdff7fd9 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 027b7dd4-77df-441b-bc74-b2a71b864d79 kind: Event lastTimestamp: "2026-07-29T10:40:44Z" message: 'Created pod: sequence-graph-f1d40-78bdff7fd9-5sdnw' metadata: creationTimestamp: "2026-07-29T10:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:44Z" name: sequence-graph-f1d40-78bdff7fd9.18c6bc93b5480341 namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: 2ee3b4fe-cb31-4716-9caf-344ce5ff5e2f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: 52e35ce2-9ab4-4595-ac97-44bc0ffe9ee8 kind: Event lastTimestamp: "2026-07-29T10:40:44Z" message: Scaled up replica set sequence-graph-f1d40-78bdff7fd9 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:44Z" name: sequence-graph-f1d40.18c6bc93b47cb539 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 49284b83-7877-4d96-b5e9-9e2d5dd9db13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: ec5f3b75-fcb6-451a-bf66-6da1179f999d kind: Event lastTimestamp: "2026-07-29T10:40:44Z" message: InferenceGraph [sequence-graph-f1d40] is Ready metadata: creationTimestamp: "2026-07-29T10:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:40:44Z" name: sequence-graph-f1d40.18c6bc93b97b1395 namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 0f5297f4-2a4c-4cf5-a11b-b1f75b979b07 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:40:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: bf3eefdc-311a-449a-9510-4f512c2fce53 kind: Event lastTimestamp: "2026-07-29T10:41:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:41:14Z" name: sequence-graph-f1d40.18c6bc9733024c29 namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: df24539e-6a33-4cfe-a859-ef8a1344230b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:40:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: bf3eefdc-311a-449a-9510-4f512c2fce53 kind: Event lastTimestamp: "2026-07-29T10:41:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:41:14Z" name: sequence-graph-f1d40.18c6bc973368d126 namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 91e86767-4e45-4d43-8858-b23121935f38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T11:00:35.429556Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-afa2d-86d55575d-mkw79 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6bda8e72c3c0d namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 2c498525-53e8-498c-b81e-63ff5ff2bdcf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6bda90112be1d namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 1de7fd6e-2851-4132-b912-b106896a8091 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afa2d} kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6bda9025de19a namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: b49817b2-4269-418f-bcef-b9c0277c0c3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afa2d} kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: 'Created container: splitter-graph-afa2d' metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6bda90571c725 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 081bed18-de9b-41bc-a477-e1f27469af9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afa2d} kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: Started container splitter-graph-afa2d metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6bda905e978f7 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 6948452c-e5fa-4a82-acb9-327887cd80c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afa2d} kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: "2026-07-29T11:08:49Z" message: Stopping container splitter-graph-afa2d metadata: creationTimestamp: "2026-07-29T11:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:08:49Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6be1c09ec9ce6 namespace: kserve-ci-e2e-test resourceVersion: "30310" uid: df40509e-756d-47c0-a349-420486b5cb8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T11:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afa2d} kind: Pod name: splitter-graph-afa2d-86d55575d-mkw79 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 057666b6-b812-479b-8970-3f6c4a5d3dd3 kind: Event lastTimestamp: "2026-07-29T11:09:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T11:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:09:17Z" name: splitter-graph-afa2d-86d55575d-mkw79.18c6be1cacfd7a98 namespace: kserve-ci-e2e-test resourceVersion: "30545" uid: c21b61ad-ed61-425b-a826-f75c9951b913 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-afa2d-86d55575d namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: ea3eb401-227c-4267-98c7-c6eebaa74d0e kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: 'Created pod: splitter-graph-afa2d-86d55575d-mkw79' metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d-86d55575d.18c6bda8e6c26131 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 5211578b-b847-4c9f-800f-995fe447d547 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-afa2d namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 7b0b14bc-d407-4b7e-8494-b8bd3b223ef4 kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: Scaled up replica set splitter-graph-afa2d-86d55575d from 0 to 1 metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d.18c6bda8e5dcfd6a namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 0ef0bda5-309d-41d5-b5ae-df4e6987e52c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-afa2d namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: aaddae2a-7974-4c22-8098-53fccc9559cd kind: Event lastTimestamp: "2026-07-29T11:00:35Z" message: InferenceGraph [splitter-graph-afa2d] is Ready metadata: creationTimestamp: "2026-07-29T11:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:35Z" name: splitter-graph-afa2d.18c6bda8eb2478aa namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: eb7ddefb-2121-4985-8173-06791514e95a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T11:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-afa2d namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: cd4f6dea-a1c2-4b84-b5c1-5e0cfc64f4b5 kind: Event lastTimestamp: "2026-07-29T11:01:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T11:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:01:05Z" name: splitter-graph-afa2d.18c6bdac6449b839 namespace: kserve-ci-e2e-test resourceVersion: "27112" uid: 2f41ff36-9aa9-4671-a678-e578d72af715 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T11:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-afa2d namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: cd4f6dea-a1c2-4b84-b5c1-5e0cfc64f4b5 kind: Event lastTimestamp: "2026-07-29T11:01:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T11:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:01:05Z" name: splitter-graph-afa2d.18c6bdac64ab62d1 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 11e5a3b8-2831-46c2-b155-0b6646ecc91f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:59:25.088836Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-f5066-5dd8b587f8-6xmdh to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd988689fd4b namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: a00df8f5-3f26-4e2a-8c75-1fd5eb2acc8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd98a07ce07f namespace: kserve-ci-e2e-test resourceVersion: "25721" uid: 11a03328-4437-4f73-93de-5fd8161f4b6c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f5066} kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd98a1bf0760 namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: 7ac328c3-d28d-495d-a814-1f319528a3b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f5066} kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'Created container: splitter-graph-f5066' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd98a4fd034c namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 94424c06-2849-4fd1-b441-f0dc176138c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f5066} kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: Started container splitter-graph-f5066 metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd98a578a103 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: af1e2597-03a2-4da4-aea8-5e27fe0d9334 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f5066} kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Stopping container splitter-graph-f5066 metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd9ae0187f25 namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: 08289cae-fbdf-4409-9ebc-6b6b8b3c6e29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f5066} kind: Pod name: splitter-graph-f5066-5dd8b587f8-6xmdh namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: c16f1604-9687-4840-9dd2-965b42fc9e7f kind: Event lastTimestamp: "2026-07-29T11:00:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:02Z" name: splitter-graph-f5066-5dd8b587f8-6xmdh.18c6bd9b63ddb9cc namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: d8928b49-dbfa-4b4a-aa03-a03c5499de95 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-f5066-5dd8b587f8 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 8894519e-a59d-4faa-94d7-5e046a6b1c70 kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'Created pod: splitter-graph-f5066-5dd8b587f8-6xmdh' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066-5dd8b587f8.18c6bd988627cd6f namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: f5027664-925c-4f4b-8871-94088dd2f1c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25679" uid: 4cc944c6-d79a-4f44-821e-1a191b0cf78b kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: Scaled up replica set splitter-graph-f5066-5dd8b587f8 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066.18c6bd988579954e namespace: kserve-ci-e2e-test resourceVersion: "25682" uid: 79d454c7-8d00-436f-b10c-2c875ede90f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: 5b715405-6c02-458c-bb7b-27dd67f01fbf kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: InferenceGraph [splitter-graph-f5066] is Ready metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066.18c6bd988a99be44 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 649e126a-e6e3-4e39-a874-53f52fdfb837 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 5b715405-6c02-458c-bb7b-27dd67f01fbf kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'Failed to update status for InferenceGraph "splitter-graph-f5066": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-f5066": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066.18c6bd988c9f05d2 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 2049b231-1fe8-49e1-960c-fbdfa64369fb reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 5b715405-6c02-458c-bb7b-27dd67f01fbf kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-f5066": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:25Z" name: splitter-graph-f5066.18c6bd988c9f5023 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 204dba0d-af9e-4faf-9c99-0c0e099ea40b reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:50:10.048302Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:50:10Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd174b8b7f85 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: a8b6adc0-574a-4cc9-ac97-be4cf8a8f2d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0934b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd17579d3015 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: dbb8b034-65a4-4902-80f7-75156ec9e1b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd17896dab9b namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 769b7543-98c0-4796-93f4-eaeb9864f093 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd178aebb489 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: b087c627-9a99-4a95-a6f7-d2b6fb99ee92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd178dd10f25 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: d8f4fee8-2f58-4009-bcf8-7d8bf590d59c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd178e4bbe18 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 7ecb6c89-5cfa-43c1-bb81-48da65c4c686 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd178e56499e namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: b9a18ef9-d3a6-4879-ae18-1da4c2e06742 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd179130cb93 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 1b6febc6-4ace-421e-889a-e396fe634a3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:50:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:11Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd1791ae6d75 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 8e28c647-7b84-4440-8309-25fa986531a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:59:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:27Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd1793f66e58 namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 6e33562b-bfe2-47bf-a460-d6d5c8b4e006 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:24Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd987a50bb43 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 8243d7a4-bd80-40d2-8a97-26109f367aae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:24Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd987a5189cd namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: cbedebf8-85bd-445e-9d86-e0e35741cf92 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: cded21d0-08a5-4c0d-916b-b27ccc2d6ccd kind: Event lastTimestamp: "2026-07-29T10:59:27Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:27Z" name: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2.18c6bd99086eea55 namespace: kserve-ci-e2e-test resourceVersion: "25765" uid: 621b7098-2cf1-46bf-8e5c-8bbf64905e24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0934b-predictor-5cb5448f5d namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 8d169d98-2fff-40d3-8642-37a00acb78d2 kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: 'Created pod: success-200-isvc-0934b-predictor-5cb5448f5d-ftmm2' metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:10Z" name: success-200-isvc-0934b-predictor-5cb5448f5d.18c6bd174b53d038 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 197d5eb4-d0a7-48f9-9e30-8b3b2a111802 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 99fae051-b28e-4106-9d9c-afaf606393bb kind: Event lastTimestamp: "2026-07-29T10:50:10Z" message: Scaled up replica set success-200-isvc-0934b-predictor-5cb5448f5d from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:10Z" name: success-200-isvc-0934b-predictor.18c6bd174952d159 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 959457b4-2d4d-4fe3-b2e1-729d768e7d35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:50:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: f2c50899-b083-468c-b1aa-19a52dddd104 kind: Event lastTimestamp: "2026-07-29T10:50:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:40Z" name: success-200-isvc-0934b-predictor.18c6bd1ac7a55e16 namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: 4adbed9f-91ca-4f58-8794-55e5a87afae2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:50:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: f2c50899-b083-468c-b1aa-19a52dddd104 kind: Event lastTimestamp: "2026-07-29T10:50:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:40Z" name: success-200-isvc-0934b-predictor.18c6bd1ac80ca611 namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: c447dc5c-7d01-4147-8ae5-622f18058738 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:50:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: f2c50899-b083-468c-b1aa-19a52dddd104 kind: Event lastTimestamp: "2026-07-29T10:51:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:25Z" name: success-200-isvc-0934b-predictor.18c6bd21c5ee3724 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: 3a069583-fb19-4bac-9a4a-a3c35c7ff92b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:50:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0934b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: f2c50899-b083-468c-b1aa-19a52dddd104 kind: Event lastTimestamp: "2026-07-29T10:51:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:51:25Z" name: success-200-isvc-0934b-predictor.18c6bd21c6563b25 namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: b4ecad6b-d289-49eb-9682-d6aebe459d37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:50:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0934b namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: e8266b8b-42c6-47a6-8ef8-d77813e73966 kind: Event lastTimestamp: "2026-07-29T10:50:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:57Z" name: success-200-isvc-0934b.18c6bd174939da80 namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: e82184f7-55ad-4ed5-9c66-d4e8e9caf4fb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0934b namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: e8266b8b-42c6-47a6-8ef8-d77813e73966 kind: Event lastTimestamp: "2026-07-29T10:50:57Z" message: InferenceService [success-200-isvc-0934b] is Ready metadata: creationTimestamp: "2026-07-29T10:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:50:57Z" name: success-200-isvc-0934b.18c6bd224efa912d namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: 178fd09d-6c33-4bce-b87c-0f4fc10ac65a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:38:44.996134Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0a743-predictor-98f985b96-6lt86 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77cb40f4a1 namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: e3507e9e-cc5f-4b83-8a9a-8263abfc8056 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77e5933b6a namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 2fe88364-c9c4-4e4f-a4be-63891eff3cfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77e710b579 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 61967ecc-1d64-40ac-9ed9-225161897625 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77eaaa9768 namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 9efa5cd1-fe87-4e62-b1cb-b6bd562a53c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77eb4b1786 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 6a005c74-3a5b-469a-ae05-ba24f56c74cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77eb5963fb namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: bcbb63d0-ec6a-4d47-b865-f654288ff3c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77edfdc0fe namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: af06a3a3-0a99-4020-8fc3-3d6a1853b4f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:38:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:45Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc77ee89aa62 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: c9355028-0b27-43a7-948a-89c0c8db59a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:48:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:02Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bc782e35d559 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 674ab14c-b44c-47d1-9bdc-9b35bec852bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:47:59Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bcf8ffbd4aad namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: c514504a-79ad-48c7-95db-a8db35fcaa42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:47:59Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bcf8ffbdae96 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 305fc9ef-aa91-49e1-bd8e-9560f45e353a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a743-predictor-98f985b96-6lt86 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 58ccf183-bc4e-47ee-a353-a4696239b953 kind: Event lastTimestamp: "2026-07-29T10:48:02Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:02Z" name: success-200-isvc-0a743-predictor-98f985b96-6lt86.18c6bcf9a296c388 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 3f08469b-5c37-4b56-888b-51c82ce02cee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0a743-predictor-98f985b96 namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: db096d15-8b05-4860-8951-a84f56013130 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: 'Created pod: success-200-isvc-0a743-predictor-98f985b96-6lt86' metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-0a743-predictor-98f985b96.18c6bc77cadac77d namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 831ae102-415b-4f85-be77-d1b2b2f38710 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 801b87f4-18ea-4cdf-83e2-2a97d978a7d9 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: Scaled up replica set success-200-isvc-0a743-predictor-98f985b96 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-0a743-predictor.18c6bc77c9410ba8 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 0527b09f-d0ee-4366-be0d-b3660fe7aa64 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 8fa50ed7-9033-4183-a904-9a4257c7c0cf kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:45Z" name: success-200-isvc-0a743-predictor.18c6bc7b47c6a3e6 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: bf9c9b2e-6778-4ccb-bda3-256c78cdc814 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 8fa50ed7-9033-4183-a904-9a4257c7c0cf kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:45Z" name: success-200-isvc-0a743-predictor.18c6bc7b483452b8 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: eebdba33-9cd1-488a-9514-6d3496cb936f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 8fa50ed7-9033-4183-a904-9a4257c7c0cf kind: Event lastTimestamp: "2026-07-29T10:39:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:14Z" name: success-200-isvc-0a743-predictor.18c6bc7ec6f160ba namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: dd230f02-6e62-410b-b991-325ee13347d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a743-predictor namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 8fa50ed7-9033-4183-a904-9a4257c7c0cf kind: Event lastTimestamp: "2026-07-29T10:39:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:15Z" name: success-200-isvc-0a743-predictor.18c6bc7ec7537267 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 59ab3459-e267-4206-bcd8-20a52b32df62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 64cb4630-32fe-4493-8417-06895f654989 kind: Event lastTimestamp: "2026-07-29T10:39:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:32Z" name: success-200-isvc-0a743.18c6bc77c9751767 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 2f26e5f9-bedc-4396-a1a5-05249cb22047 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 64cb4630-32fe-4493-8417-06895f654989 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-0a743": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0a743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-0a743.18c6bc77c9cefee9 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 5d8493d5-eb1c-4e08-8d4c-78e88c5c8359 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 64cb4630-32fe-4493-8417-06895f654989 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0a743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-0a743.18c6bc77c9cf708b namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: e45efc36-bb10-4bac-ad18-4d2235f63226 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a743 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 64cb4630-32fe-4493-8417-06895f654989 kind: Event lastTimestamp: "2026-07-29T10:39:32Z" message: InferenceService [success-200-isvc-0a743] is Ready metadata: creationTimestamp: "2026-07-29T10:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:32Z" name: success-200-isvc-0a743.18c6bc82e90a5133 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bd61be84-0a97-4276-bd8f-1b64377156c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:48:59.775329Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6242a-predictor-cbf7db67b-9cr5q to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd06eef30b6b namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: 55eee1db-738f-419d-be27-e3032421e1c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6242a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd06f6202f29 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: c1d6dc73-5b9c-4224-81b1-dd4a4835f367 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd072cccba21 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 0561f727-9f84-422b-97fd-41e496bc2585 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd072e45a8b0 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: cd822195-cdbb-4923-b4e8-c8fc9d0d40b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd07311e76fb namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 6e14ee32-6e67-4c16-b00c-b0b622d8123f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd0731980903 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: c4e4838f-78f2-4a92-88ab-a20228caca1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd0731a27d06 namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: 5732ed0e-3900-42fb-bebc-a237b61d66da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd07349e9223 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 7fed36e9-aa7b-4b5d-8b18-4f07f46e3ae0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:00Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd07351efc26 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 8600e602-0d87-4bdd-885f-785c02ba5250 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:49:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:37Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd0771fe6c3e namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 87de1a54-3a80-432c-882c-936817c3e660 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:50:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:09Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd17487a2e87 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 190bb80d-1591-469b-813c-304d243608ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:50:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:10Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd17487ac8d0 namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 5882685f-9db5-46d1-ac93-1ccda368bddf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0afd87a2-5d84-4854-bc44-6835c5be75cb kind: Event lastTimestamp: "2026-07-29T10:50:12Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:50:12Z" name: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q.18c6bd17fa49be6c namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 66f095c4-d942-47b2-a34e-be3adc954e64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6242a-predictor-cbf7db67b namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 1b381838-3fbc-4132-abd5-c8e39cfc97a3 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'Created pod: success-200-isvc-6242a-predictor-cbf7db67b-9cr5q' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-6242a-predictor-cbf7db67b.18c6bd06ee8abc9a namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 7757f561-fc79-476c-906c-fa291419970a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: d1998582-8d0b-4ec2-a6a9-2d1bd7dee809 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Scaled up replica set success-200-isvc-6242a-predictor-cbf7db67b from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-6242a-predictor.18c6bd06ec7a84bb namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 2e8cb245-d562-47cd-8ca3-1a8199ad746f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 9108516a-d6f8-40e0-bac2-97ee1fad9115 kind: Event lastTimestamp: "2026-07-29T10:49:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:29Z" name: success-200-isvc-6242a-predictor.18c6bd0a6b3dba55 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: aea63302-73b9-4694-88d5-98742891cceb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 9108516a-d6f8-40e0-bac2-97ee1fad9115 kind: Event lastTimestamp: "2026-07-29T10:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:29Z" name: success-200-isvc-6242a-predictor.18c6bd0a6bae320f namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 1f0b1e91-d0dc-4853-b2bb-f5aac3419c30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:49:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 9108516a-d6f8-40e0-bac2-97ee1fad9115 kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:59Z" name: success-200-isvc-6242a-predictor.18c6bd11692ab97b namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 49fde450-0f91-4fda-98b0-d98297916638 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:49:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6242a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 9108516a-d6f8-40e0-bac2-97ee1fad9115 kind: Event lastTimestamp: "2026-07-29T10:49:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:59Z" name: success-200-isvc-6242a-predictor.18c6bd116998a15f namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 3d8a158f-b243-4b0c-ab59-05d519de36d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 35cc9239-294f-41b4-a188-f398b318c863 kind: Event lastTimestamp: "2026-07-29T10:49:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:48Z" name: success-200-isvc-6242a.18c6bd06eca51264 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: eb104701-ebee-4b19-a04d-84a2ebd424a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 35cc9239-294f-41b4-a188-f398b318c863 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-6242a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6242a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-6242a.18c6bd06ed71e803 namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: c8486ba3-a051-45f5-9116-be6d230c42ce reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 35cc9239-294f-41b4-a188-f398b318c863 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6242a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-6242a.18c6bd06ed7241fd namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 4a23eb5c-1ab0-4681-a46e-9d8c858a3f1a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6242a namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 35cc9239-294f-41b4-a188-f398b318c863 kind: Event lastTimestamp: "2026-07-29T10:49:48Z" message: InferenceService [success-200-isvc-6242a] is Ready metadata: creationTimestamp: "2026-07-29T10:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:48Z" name: success-200-isvc-6242a.18c6bd122d1ba97c namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 506f1480-b22d-4905-a922-2b85750686cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:59:24.947851Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:59:24Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd987e22bf33 namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: bbe328c5-6e1b-4e1f-bcd1-17468e446f34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-965a2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:25Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd988e24829e namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 13856103-ca43-4981-9be0-516cbc4fcad5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:59:25Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98bc0da1ee namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: d25c9d11-a174-4122-9059-446a07c90725 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98bd8c6ba3 namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: f0aa351d-499c-4dc9-adf4-2578ec978184 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98c0af40e1 namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: bc92e2dc-579f-4981-86f6-dbebe58f0847 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98c129f9bf namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: 653ceb6f-3528-42fb-8864-3b09a4fe0791 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98c135fd7c namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: b27786e2-aa81-4a3f-8597-23ddbb6778e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98c53c66a5 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 8073bd89-0425-47c3-b6db-27febf9d3959 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T10:59:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:26Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd98c5bc5bb0 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 55b4e86f-19dc-40a2-8041-1938c1fac1f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T11:00:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:03Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6bd990fa893e1 namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: e56c8baf-fdb2-49b1-90ce-7d9c52b50dfa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T11:16:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T11:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:16:44Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6be8a8a9a875b namespace: kserve-ci-e2e-test resourceVersion: "33619" uid: a628b9e4-1ffb-4ba9-b60e-fb4e4015c2c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 43d2a27d-baba-45b7-a758-cbe3a4ee264c kind: Event lastTimestamp: "2026-07-29T11:16:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T11:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:16:44Z" name: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz.18c6be8a8a9bbde0 namespace: kserve-ci-e2e-test resourceVersion: "33620" uid: fdc6ee54-7d18-4b54-b050-e29dcf1a10dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-965a2-predictor-64fd5c48d4 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: e8375912-ad3b-4aa0-8b9a-01d42d167acb kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: 'Created pod: success-200-isvc-965a2-predictor-64fd5c48d4-rjkhz' metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:24Z" name: success-200-isvc-965a2-predictor-64fd5c48d4.18c6bd987dbe1a9c namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: 0414840d-e592-46ab-9b99-085242ac4e5d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 7c8bf264-4d57-415e-9c08-39e1abef43cd kind: Event lastTimestamp: "2026-07-29T10:59:24Z" message: Scaled up replica set success-200-isvc-965a2-predictor-64fd5c48d4 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:24Z" name: success-200-isvc-965a2-predictor.18c6bd987b12d819 namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: 37d2abf4-8bbb-4bfb-839f-481221768c77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25608" uid: 1cdbec0f-9f46-43f9-92e3-b51e40e7c19d kind: Event lastTimestamp: "2026-07-29T10:59:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:54Z" name: success-200-isvc-965a2-predictor.18c6bd9bfa203409 namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: b118de01-f10f-4478-ab48-390c5a251a02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25608" uid: 1cdbec0f-9f46-43f9-92e3-b51e40e7c19d kind: Event lastTimestamp: "2026-07-29T10:59:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:54Z" name: success-200-isvc-965a2-predictor.18c6bd9bfa940236 namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: e517941c-a8ee-4883-a049-be7d7131e737 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T11:00:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: 1cdbec0f-9f46-43f9-92e3-b51e40e7c19d kind: Event lastTimestamp: "2026-07-29T11:00:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T11:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:39Z" name: success-200-isvc-965a2-predictor.18c6bda2f88496da namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 9362faad-bfff-4652-b6bd-063e6efd9091 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T11:00:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-965a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: 1cdbec0f-9f46-43f9-92e3-b51e40e7c19d kind: Event lastTimestamp: "2026-07-29T11:00:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T11:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:39Z" name: success-200-isvc-965a2-predictor.18c6bda2f8eeb61f namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: efbd816d-c7ce-497d-9013-2df167223b1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-965a2 namespace: kserve-ci-e2e-test resourceVersion: "25588" uid: c31b4dd9-632b-44da-bdff-216c743f616c kind: Event lastTimestamp: "2026-07-29T11:00:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:13Z" name: success-200-isvc-965a2.18c6bd987b156908 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 05dba3e1-7b54-4160-a922-94cfb6ede8fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-965a2 namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: c31b4dd9-632b-44da-bdff-216c743f616c kind: Event lastTimestamp: "2026-07-29T11:00:13Z" message: InferenceService [success-200-isvc-965a2] is Ready metadata: creationTimestamp: "2026-07-29T11:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:13Z" name: success-200-isvc-965a2.18c6bda3cab4fee3 namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: 187de2f5-3efe-4e52-aa19-bbb621cc7e07 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:59:35.350217Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9aea2a3112 namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: 89e7cd25-12ec-4224-ad3d-60283d9cf599 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-afa2d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9af8ff3c8b namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 6e3e38bc-69f5-444c-826b-6b3638670379 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25917" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b28bae7c2 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 819d0036-b972-427c-b580-0942b8e084bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b2a21212f namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: e9db1644-b313-44d9-a9b6-a0f1b5df689d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b2d4c2c11 namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 7dc1458b-3c84-4613-8c1b-e6388c7c291d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b2dc524c8 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: 203fcedb-4b50-4a72-b878-6cba03de7a2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b2dcfc754 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: 2d25c5a0-cbd3-44b2-aaba-31790155bb43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b31b4d611 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 37512601-da3a-4a54-88f5-12be8b6c7eb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T10:59:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:36Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b322ef9a0 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 9c691b7f-f499-47e7-b0a9-7f38290925c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T11:00:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:13Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6bd9b663dfb1c namespace: kserve-ci-e2e-test resourceVersion: "26601" uid: 53e976f6-c118-4f5b-84cb-65af0a9c1a04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T11:08:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T11:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:08:50Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6be1c11e81f29 namespace: kserve-ci-e2e-test resourceVersion: "30327" uid: f1e3c2ac-204f-4885-9717-6e339a46074d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 91dea421-e029-486a-a81b-cc4e6f1b0a60 kind: Event lastTimestamp: "2026-07-29T11:08:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T11:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:08:50Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2.18c6be1c11e8b255 namespace: kserve-ci-e2e-test resourceVersion: "30328" uid: 6028b539-37dc-466c-8ba2-973aa654b8d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-afa2d-predictor-54c8dbf4d6 namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 9a78acd1-d684-4882-8a34-c2ea47e951ef kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'Created pod: success-200-isvc-afa2d-predictor-54c8dbf4d6-k86d2' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-afa2d-predictor-54c8dbf4d6.18c6bd9ae9c07289 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 9192aadc-32b1-4adc-b6fc-948f97381907 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 34a529d9-1840-4b59-9aba-ff1176dcf748 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Scaled up replica set success-200-isvc-afa2d-predictor-54c8dbf4d6 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-afa2d-predictor.18c6bd9ae80299bc namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: 36e836af-0fc3-49e5-86c1-84570aed30db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25894" uid: 70d3a71d-cff3-4c8f-8bf8-371b62d5de37 kind: Event lastTimestamp: "2026-07-29T11:00:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:05Z" name: success-200-isvc-afa2d-predictor.18c6bd9e66bfde0a namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 71138c3c-5863-4018-bf61-bbcf10103c32 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25894" uid: 70d3a71d-cff3-4c8f-8bf8-371b62d5de37 kind: Event lastTimestamp: "2026-07-29T11:00:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:05Z" name: success-200-isvc-afa2d-predictor.18c6bd9e672aa4d1 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: b0a3a1c9-e2ee-4db6-9ae3-7da5007d5a20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T11:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 70d3a71d-cff3-4c8f-8bf8-371b62d5de37 kind: Event lastTimestamp: "2026-07-29T11:00:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T11:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:50Z" name: success-200-isvc-afa2d-predictor.18c6bda564adb11a namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: dc290b48-b453-4d05-bbc3-e3dfb4d2455f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T11:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afa2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 70d3a71d-cff3-4c8f-8bf8-371b62d5de37 kind: Event lastTimestamp: "2026-07-29T11:00:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T11:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:50Z" name: success-200-isvc-afa2d-predictor.18c6bda56529987e namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: 740abe9c-3e8a-413f-914f-75ab74dc91bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: 6c278626-87f7-437e-95f3-7e92d1379300 kind: Event lastTimestamp: "2026-07-29T11:00:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:23Z" name: success-200-isvc-afa2d.18c6bd9ae82e5a8e namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: 23e539d0-b27f-4155-8933-8d3d1919cc39 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: 6c278626-87f7-437e-95f3-7e92d1379300 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-afa2d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-afa2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-afa2d.18c6bd9ae8a32107 namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: fd337901-727d-4867-8e46-6361ee842e89 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: 6c278626-87f7-437e-95f3-7e92d1379300 kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-afa2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-afa2d.18c6bd9ae8a37766 namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: cefdbb71-436c-4470-8a5e-139571fd1ea4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afa2d namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: 6c278626-87f7-437e-95f3-7e92d1379300 kind: Event lastTimestamp: "2026-07-29T11:00:23Z" message: InferenceService [success-200-isvc-afa2d] is Ready metadata: creationTimestamp: "2026-07-29T11:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:23Z" name: success-200-isvc-afa2d.18c6bda620edf6c2 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: bec95da1-8035-46dc-b7a3-51dc780e9ad9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:47:59.963532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:47:59Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf901e36bfd namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 24af87a7-9144-4897-b3f0-e72956bb0504 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf91c0daec1 namespace: kserve-ci-e2e-test resourceVersion: "19836" uid: 141f2b94-badd-4725-be26-03588150fad4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf91d6a0e61 namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: 77956878-d751-449f-a0be-2c33d54165ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf92070019c namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: 8f4d0497-9e1d-4702-8f09-a4bbf1777fb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf920ec943a namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: d8b01c58-84f4-4db7-a17c-55cea1f31e33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf920f7f4dc namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 3286f013-012e-4685-aa71-f9a1b548b03a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf925319289 namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 946a9947-3465-4886-bf0c-9b52c41451d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:00Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf925ab18af namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: ee95898b-bd62-4ab9-aa9f-5e46185e4ab5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:48:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:36Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bcf92f5fed3f namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: 48422929-e201-4eb2-af15-87bcbb8cccf8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bd095eb549ac namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 44d92c1d-5c9a-4f02-baaa-cc83fee6524e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bd095eb5de17 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: a3cda4ba-16fe-4dbd-92f0-51b59d9b99e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: b6cca808-2600-4e20-a498-fd3ea5b3c3ce kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw.18c6bd09b7778451 namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: bacec876-dcc1-409a-8005-888c3ae1d235 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b12e8-predictor-57c478d4fd namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c57df915-cf6c-4bae-b8d6-7ac72c31c9cd kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: 'Created pod: success-200-isvc-b12e8-predictor-57c478d4fd-lwqkw' metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:47:59Z" name: success-200-isvc-b12e8-predictor-57c478d4fd.18c6bcf90177a19b namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 3c37cb73-d188-476f-8dce-58f8e9c5ce7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: c36f0360-4363-4f8e-bb36-5cd41b5d3748 kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: Scaled up replica set success-200-isvc-b12e8-predictor-57c478d4fd from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:47:59Z" name: success-200-isvc-b12e8-predictor.18c6bcf8fe07808f namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: a5416abe-3eb4-4b1b-a390-524e308974a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: dfb70cb5-5deb-46f5-9f7a-c6b1e117e996 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-b12e8-predictor.18c6bcfc7d1f5b57 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 1a0a8686-fa55-4ead-a202-eee24066ecb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b12e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: dfb70cb5-5deb-46f5-9f7a-c6b1e117e996 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-b12e8-predictor.18c6bcfc7d8b6915 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 25ce5a5a-2351-4f0e-9094-d20f77717df7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 2557469c-8a4a-4eb7-b099-0be2790759aa kind: Event lastTimestamp: "2026-07-29T10:48:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:46Z" name: success-200-isvc-b12e8.18c6bcf8fe157cf1 namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 5f9306ce-2388-408f-a3b3-1055427f85a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b12e8 namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 2557469c-8a4a-4eb7-b099-0be2790759aa kind: Event lastTimestamp: "2026-07-29T10:48:46Z" message: InferenceService [success-200-isvc-b12e8] is Ready metadata: creationTimestamp: "2026-07-29T10:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:48:46Z" name: success-200-isvc-b12e8.18c6bd03ea2a5190 namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 5bec2ba2-97bd-48cc-bbe8-9f724f23e044 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:49:10.259455Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d2d23-predictor-77b7bd766d-vx77s to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd095fda175f namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: aaf734d0-7a3d-4e82-9f2e-d850c84c86c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d2d23-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd0966e76274 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 1d020fe9-73df-458a-a6cd-64c2fe3e706d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd099de1e8fc namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 021347de-a29d-4165-9245-97ab0ec648b1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd099f3f5473 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: 68fd095a-a529-4874-9b61-de2d056b2dd0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd09a2663648 namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: 618ec50e-da13-4e9e-adfa-267f06123db0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd09a2e1ab69 namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 53e1a915-594a-4828-bd51-9f87eb765b61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd09a2ec8182 namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 2d22021b-20f7-436d-9b25-d2d7bef80a74 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd09a5f70694 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: 5d908746-3c76-4c1f-bfda-79c93b209830 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:49:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:49:11Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd09a6850823 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 675e46ad-a917-4767-8872-c316e348f936 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-29T10:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:58:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:28Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd09c8cc6a00 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: 0a297b73-2ae1-48a7-b0f5-ec48e4512e44 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd8a8fb52697 namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: 12f74b4e-44fa-4122-b4ca-c51b18f579e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd8a8fb63ec0 namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: cc4320d9-0f19-42d9-8f28-b92974ec3e05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 44d53f76-1981-4b72-b824-2c949abdf491 kind: Event lastTimestamp: "2026-07-29T10:58:28Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:28Z" name: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s.18c6bd8b3d82734d namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: 75ac163a-2417-4e57-9aea-c4a8ba1465f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d2d23-predictor-77b7bd766d namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 66fc310f-cce6-415b-a5a0-eff1b24c0737 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'Created pod: success-200-isvc-d2d23-predictor-77b7bd766d-vx77s' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-d2d23-predictor-77b7bd766d.18c6bd095f3ecbb5 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 5d71b335-6a88-434a-8a3d-76553266c679 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: b1c3b20b-5246-4c49-b5b7-2f8a224881d4 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: Scaled up replica set success-200-isvc-d2d23-predictor-77b7bd766d from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-d2d23-predictor.18c6bd095d67a614 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7544742-8775-46c8-9857-3e5d845c45e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: f6b70d9d-47a3-4e55-b272-056cbfc4e1d6 kind: Event lastTimestamp: "2026-07-29T10:49:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:25Z" name: success-200-isvc-d2d23-predictor.18c6bd0cdbeda20e namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 53452f99-076b-4686-844e-36ad8f1d0e35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: f6b70d9d-47a3-4e55-b272-056cbfc4e1d6 kind: Event lastTimestamp: "2026-07-29T10:49:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:49:25Z" name: success-200-isvc-d2d23-predictor.18c6bd0cdc55366b namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 95776495-aa2d-4486-a7ea-030bceb92601 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:49:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: f6b70d9d-47a3-4e55-b272-056cbfc4e1d6 kind: Event lastTimestamp: "2026-07-29T10:50:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:25Z" name: success-200-isvc-d2d23-predictor.18c6bd105b7479d9 namespace: kserve-ci-e2e-test resourceVersion: "21609" uid: 951dfeaa-ce9f-43e1-85c5-be69b277077e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:49:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: f6b70d9d-47a3-4e55-b272-056cbfc4e1d6 kind: Event lastTimestamp: "2026-07-29T10:50:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:50:25Z" name: success-200-isvc-d2d23-predictor.18c6bd105be81877 namespace: kserve-ci-e2e-test resourceVersion: "21610" uid: 65e7a8ed-36bd-4f11-bed2-4ec43e70a4e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: d78c820f-7307-4869-9db8-c61346be27f1 kind: Event lastTimestamp: "2026-07-29T10:49:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:58Z" name: success-200-isvc-d2d23.18c6bd095d8c1cf7 namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: b203c2d7-30fb-4b87-bfcb-18847cde853e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: d78c820f-7307-4869-9db8-c61346be27f1 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-d2d23": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d2d23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-d2d23.18c6bd095e4d9ba1 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: e7ef20b6-e040-4af9-b64c-54824802179a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: d78c820f-7307-4869-9db8-c61346be27f1 kind: Event lastTimestamp: "2026-07-29T10:49:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d2d23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:10Z" name: success-200-isvc-d2d23.18c6bd095e4dfd6a namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 91e9312b-f0fa-405b-b404-44fa95f88fe4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:49:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d23 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: d78c820f-7307-4869-9db8-c61346be27f1 kind: Event lastTimestamp: "2026-07-29T10:49:58Z" message: InferenceService [success-200-isvc-d2d23] is Ready metadata: creationTimestamp: "2026-07-29T10:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:49:58Z" name: success-200-isvc-d2d23.18c6bd1484dd15dd namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 9aca9472-9796-4048-b847-1d246e860b29 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:37:14.830671Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:37:14Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc62ccf93660 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 650319b0-636d-4071-a778-79aca1b46872 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: Add eth0 [10.134.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:37:15Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc62e8d7c844 namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 98574b3c-26a7-47c2-97a1-f90de21465f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:15Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" metadata: creationTimestamp: "2026-07-29T10:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:15Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc62ea5550b9 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: d67dd16b-150a-49b1-b16f-4c33fe13ac56 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:29Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" in 13.742s (13.742s including waiting). Image size: 1336828526 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:29Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc661d6c486a namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 9e94c7a6-8353-4758-96ed-68df00301829 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:29Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc6620e50192 namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: f9b42974-9ce4-4c83-8eba-8e29eee9b3a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:29Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc662165c8ad namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: 3116a737-3ea9-4098-a1e2-3116377336b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:29Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc6621747142 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: de692d06-d6e1-46ff-87f7-2ea17b6e8215 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.251s (2.251s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-29T10:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:31Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc66a7aa2eef namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 21793ecf-c166-4f01-8ad3-36fdb037fc08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:31Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc66ab5eb5c1 namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 57f5b200-4881-4f4c-81f1-4c53efcee31f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:37:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:37:31Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc66abed0bd6 namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 2a6d3066-a744-4647-b884-d9de2ea2e96a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:37:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:38:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:37:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:08Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc66e22c8c72 namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 52541cc6-e774-4c50-8cfa-1fa99290c8a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc77c5e89a05 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 6cea6306-e737-4daf-b436-f1f23b6362ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: fcb7ffb2-19af-4e74-b44b-1dbb70855951 kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx.18c6bc77c5e9d151 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: c4caad73-7868-4991-93e1-682534b47516 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f0182-predictor-54bbdb7d5b namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: c70f2cdc-7a1f-4d06-ba05-7da8f25e5f32 kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: 'Created pod: success-200-isvc-f0182-predictor-54bbdb7d5b-vj5fx' metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: success-200-isvc-f0182-predictor-54bbdb7d5b.18c6bc62cc8f9092 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: efbac9fa-2b6f-4507-9fad-e77aab113aff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 9c146085-d7c5-47e7-9888-195f3a90d986 kind: Event lastTimestamp: "2026-07-29T10:37:14Z" message: Scaled up replica set success-200-isvc-f0182-predictor-54bbdb7d5b from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:14Z" name: success-200-isvc-f0182-predictor.18c6bc62c9782f92 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 023b5438-5da4-4e65-9867-18ff3a14fb62 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 7e2db6e0-b4a8-45e6-8e3d-ff06b5233ede kind: Event lastTimestamp: "2026-07-29T10:37:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:59Z" name: success-200-isvc-f0182-predictor.18c6bc66480b25ff namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: f0fbe838-4516-4299-a1e5-67da88c350ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 7e2db6e0-b4a8-45e6-8e3d-ff06b5233ede kind: Event lastTimestamp: "2026-07-29T10:37:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:37:59Z" name: success-200-isvc-f0182-predictor.18c6bc664869b46d namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 1e1e6575-cfe1-49e5-b10f-fc9f34fcd84a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 7e2db6e0-b4a8-45e6-8e3d-ff06b5233ede kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-f0182-predictor.18c6bc69c713ef7c namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 18dd19c5-3c0d-464e-8082-0fb50511367b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0182-predictor namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 7e2db6e0-b4a8-45e6-8e3d-ff06b5233ede kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:44Z" name: success-200-isvc-f0182-predictor.18c6bc69c77d6da0 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: 356b9c47-215c-4b52-be9d-f8857466c8b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-29T10:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0182 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 6aeec08f-fd97-4c2d-9cf0-a46dc59e7cfa kind: Event lastTimestamp: "2026-07-29T10:38:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:18Z" name: success-200-isvc-f0182.18c6bc62c9924bc3 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: da2e340d-fa05-4d37-b573-2f4efb0e915a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0182 namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 6aeec08f-fd97-4c2d-9cf0-a46dc59e7cfa kind: Event lastTimestamp: "2026-07-29T10:38:18Z" message: InferenceService [success-200-isvc-f0182] is Ready metadata: creationTimestamp: "2026-07-29T10:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:18Z" name: success-200-isvc-f0182.18c6bc719f3a7f08 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 4187ada5-572f-4413-8f6c-fe99806e17b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:39:44.891851Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f1d40-predictor-58f666b764-89zbs to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:39:44Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc85bd511517 namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 4c52ec23-13ee-4edb-9a8d-b7a9edf247d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f1d40-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc85cb1af644 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 2575db06-9ba8-4640-9c81-04311d783ce2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc8631a7691a namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 0c083ab6-e170-42fa-84e0-4d3ff9f60739 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc8633083073 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 4b4bc5b0-09e9-4e90-9577-e2395dce4427 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc86363a7b99 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 068d9ead-f08e-4a12-b85d-6b1f366f519c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc8636b4d9c1 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 34fa1230-3990-4f0e-ac7c-f83f24ce0850 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc8636c01c1c namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: c56c4313-c83f-4a27-bdfc-ad3d892b4519 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc863a05ff42 namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 4c1f4598-a61e-4472-b12f-216f81d81f86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc863a8099e0 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 9123bb37-e40c-4e52-b1a8-44127f71bb7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:40:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:40:23Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bc867057d2d6 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: b8c2fe4b-bbd4-4ce5-ba75-8a989fc96e40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bd06ec763f06 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: d35fdd45-ee5d-4ac9-b46f-cb8724654be4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f1d40-predictor-58f666b764-89zbs namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d7ca144f-8def-4bac-a45f-89d3695e9928 kind: Event lastTimestamp: "2026-07-29T10:48:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:59Z" name: success-200-isvc-f1d40-predictor-58f666b764-89zbs.18c6bd06ec7666de namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 74cbe664-a03d-4833-8fd1-e102d927a35c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f1d40-predictor-58f666b764 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: f329306a-f1dd-4c59-9f15-5317004b1e89 kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: 'Created pod: success-200-isvc-f1d40-predictor-58f666b764-89zbs' metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:44Z" name: success-200-isvc-f1d40-predictor-58f666b764.18c6bc85bcd85ba5 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 2359cbad-8bf8-42a5-a03d-7d43f5e6dff2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 8deae0c0-2b2a-4e43-a01a-fecac44e675f kind: Event lastTimestamp: "2026-07-29T10:39:44Z" message: Scaled up replica set success-200-isvc-f1d40-predictor-58f666b764 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:44Z" name: success-200-isvc-f1d40-predictor.18c6bc85baf679b9 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: f7d84328-2d7e-422e-9678-2f1b11f9c9ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 46ecbd5d-72d4-4ff9-9f78-2618b26e8fe7 kind: Event lastTimestamp: "2026-07-29T10:40:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:59Z" name: success-200-isvc-f1d40-predictor.18c6bc8939ad024b namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 77d0dab0-db8d-450b-9a34-397ce64f4cc1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-29T10:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 46ecbd5d-72d4-4ff9-9f78-2618b26e8fe7 kind: Event lastTimestamp: "2026-07-29T10:40:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:59Z" name: success-200-isvc-f1d40-predictor.18c6bc893a179c9f namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 3999c759-74c7-4233-8b2d-af44bac2b0e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 46ecbd5d-72d4-4ff9-9f78-2618b26e8fe7 kind: Event lastTimestamp: "2026-07-29T10:40:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:14Z" name: success-200-isvc-f1d40-predictor.18c6bc8cb94f699a namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: a0b06c8c-14f5-44fe-a27f-f50ec2d46af8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f1d40-predictor namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 46ecbd5d-72d4-4ff9-9f78-2618b26e8fe7 kind: Event lastTimestamp: "2026-07-29T10:40:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:14Z" name: success-200-isvc-f1d40-predictor.18c6bc8cb9b7a384 namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: a1f84cbc-b0ff-4ced-88a1-0ab62cf8f87b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: dfc399ec-9725-45f2-ba3d-4d9078f252fd kind: Event lastTimestamp: "2026-07-29T10:40:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:40:34Z" name: success-200-isvc-f1d40.18c6bc85bb237730 namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 5396d8b9-b149-4eb7-8554-250db1b62373 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:40:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f1d40 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: dfc399ec-9725-45f2-ba3d-4d9078f252fd kind: Event lastTimestamp: "2026-07-29T10:40:33Z" message: InferenceService [success-200-isvc-f1d40] is Ready metadata: creationTimestamp: "2026-07-29T10:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:40:33Z" name: success-200-isvc-f1d40.18c6bc912b28210a namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 467d384d-d11d-4830-82b6-60f6800f5c95 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:58:25.107127Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f5066-predictor-569b8fdd8-tgl8h to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8a8f5a2d65 namespace: kserve-ci-e2e-test resourceVersion: "25019" uid: 50262a25-5178-41ac-b7d0-c8d7e3796510 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25026" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8aa9d05dc8 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: c69fc170-4e6a-4ad8-a2d3-bcaed36e986a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1828" already present on machine metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8aab4f7ed9 namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: 21fe892a-031e-499c-8985-6504ed5582dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8aae45fae3 namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: 0dbe8e6a-9228-461d-856a-91a3ec72c914 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8aaec46523 namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 4cdd85b7-e235-4643-9896-aea89e1fbd85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8aaecfdd5a namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: a7ea0b8a-8b19-4430-83cf-5cec2a01062c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8ab24cb23e namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: dd30e480-2562-4cd8-a5bb-015abcca4196 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8ab2fdd080 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: d0ff67e0-d911-4c29-a2cb-c48467e85b1f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:58:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:59:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:58:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:02Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd8accd112c9 namespace: kserve-ci-e2e-test resourceVersion: "25385" uid: 6c39bd8c-3ed6-4f17-93c9-27adb8c4ba5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd9ae88efd27 namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: bc0eded5-5da1-4bcb-9836-922357d47bbd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:59:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-29T10:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:35Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd9ae88f95b0 namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: d69d4bc6-08c6-44e3-b093-2401a6c2d3bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 159f0b39-a911-48cc-b0a2-8034742e990a kind: Event lastTimestamp: "2026-07-29T10:59:37Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-29T10:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:59:37Z" name: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h.18c6bd9b552420c2 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 165cab5e-58f5-410c-b5ee-cee526fd4729 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f5066-predictor-569b8fdd8 namespace: kserve-ci-e2e-test resourceVersion: "24996" uid: ad39649a-40bb-4a0d-bb33-8edd5b1e547f kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: 'Created pod: success-200-isvc-f5066-predictor-569b8fdd8-tgl8h' metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor-569b8fdd8.18c6bd8a8ee662f9 namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: fdf750a8-d819-4e6d-b6db-6ba59445971e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "24994" uid: 06f3ebe0-1ea5-4619-ace4-3a60b8566b32 kind: Event lastTimestamp: "2026-07-29T10:58:25Z" message: Scaled up replica set success-200-isvc-f5066-predictor-569b8fdd8 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:25Z" name: success-200-isvc-f5066-predictor.18c6bd8a8b18f6bd namespace: kserve-ci-e2e-test resourceVersion: "24998" uid: b4c07633-f0f3-415d-ac82-9de82a8d053f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: 0e1ff1c5-13fe-4219-b71f-433688a46139 kind: Event lastTimestamp: "2026-07-29T10:58:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:40Z" name: success-200-isvc-f5066-predictor.18c6bd8e0a0cf38f namespace: kserve-ci-e2e-test resourceVersion: "25214" uid: 13fbffed-90cf-4ba3-a17a-e3d391cc9842 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:58:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: 0e1ff1c5-13fe-4219-b71f-433688a46139 kind: Event lastTimestamp: "2026-07-29T10:58:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:58:40Z" name: success-200-isvc-f5066-predictor.18c6bd8e0a54b77d namespace: kserve-ci-e2e-test resourceVersion: "25216" uid: aedfd5b8-2ad2-4cb5-8274-5c8e8ab7fcc4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: 0e1ff1c5-13fe-4219-b71f-433688a46139 kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: success-200-isvc-f5066-predictor.18c6bd918905bae2 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 2cb73aa0-18e1-4d7c-b916-7c2b504f8048 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-29T10:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f5066-predictor namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: 0e1ff1c5-13fe-4219-b71f-433688a46139 kind: Event lastTimestamp: "2026-07-29T10:59:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-29T10:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:59:25Z" name: success-200-isvc-f5066-predictor.18c6bd91894dffa8 namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: c483ca70-72c4-49c7-8660-63552c2ceea5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:58:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f5066 namespace: kserve-ci-e2e-test resourceVersion: "24986" uid: 63097a5a-3a92-4db8-bf0e-b83e68acfdeb kind: Event lastTimestamp: "2026-07-29T10:59:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-29T10:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:12Z" name: success-200-isvc-f5066.18c6bd8a8b235707 namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 83d6d672-9923-44f8-927f-ed28e60254c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:59:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f5066 namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 63097a5a-3a92-4db8-bf0e-b83e68acfdeb kind: Event lastTimestamp: "2026-07-29T10:59:12Z" message: InferenceService [success-200-isvc-f5066] is Ready metadata: creationTimestamp: "2026-07-29T10:59:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:59:12Z" name: success-200-isvc-f5066.18c6bd9587db901a namespace: kserve-ci-e2e-test resourceVersion: "25468" uid: 5ee2f136-0ee7-48b9-a822-f894f7ee30a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:39:45.071652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-0a743-9d56587d4-sqlmx to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:39:45Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bc85c8089e3d namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 4fb5fd26-6e0c-4c4c-91c8-3da87c52aeee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-0a743-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:45Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bc85d11e385a namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: aed63dd1-4949-44d6-8068-6aa552117711 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:39:46Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bc860635f523 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: bb857313-864b-49b2-8e18-e4a4a22e27a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0a743} kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bc8608228308 namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: 7e582868-c307-4cf9-9650-c5dcb2afa0fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0a743} kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: 'Created container: switch-graph-0a743' metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bc860b3d6d36 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 94fd52a8-51de-4f8c-8720-f8ea51658cc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0a743} kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:39:46Z" message: Started container switch-graph-0a743 metadata: creationTimestamp: "2026-07-29T10:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:46Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bc860bb5d941 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 6d394495-8fbb-4f25-ab42-d95646cb375f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0a743} kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:47:59Z" message: Stopping container switch-graph-0a743 metadata: creationTimestamp: "2026-07-29T10:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:47:59Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bcf8f676be99 namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: 64411ba8-ed23-4d72-9c0a-73463110839c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0a743} kind: Pod name: switch-graph-0a743-9d56587d4-sqlmx namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 4d991797-5a7c-439d-af61-5d7f08d8b15a kind: Event lastTimestamp: "2026-07-29T10:48:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:48:27Z" name: switch-graph-0a743-9d56587d4-sqlmx.18c6bcf9b0d25b41 namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: 3f9284ea-f761-496d-a6a4-e42137f4e487 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-0a743-9d56587d4 namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: f0d3f37e-5253-4479-a1a2-3d1955c4287c kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: 'Created pod: switch-graph-0a743-9d56587d4-sqlmx' metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:45Z" name: switch-graph-0a743-9d56587d4.18c6bc85c7aa719f namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 1db22fbe-a949-4758-9eb4-1675e53fc391 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-0a743 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: c9dc4e8d-4043-4d4a-a401-88c16d6dc153 kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: Scaled up replica set switch-graph-0a743-9d56587d4 from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:39:45Z" name: switch-graph-0a743.18c6bc85c70833fa namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 4931a2df-e6c9-41e5-b33e-e6983208ebeb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:39:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-0a743 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 35ea2774-1751-406d-99a8-2e1336f1f10f kind: Event lastTimestamp: "2026-07-29T10:39:45Z" message: InferenceGraph [switch-graph-0a743] is Ready metadata: creationTimestamp: "2026-07-29T10:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:39:45Z" name: switch-graph-0a743.18c6bc85ccec5ba9 namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: 9d51b077-5d3d-4b3e-9785-e6b7c7093321 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0a743 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 24190e2b-33d7-4b88-a650-22cde8abe7a1 kind: Event lastTimestamp: "2026-07-29T10:40:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:15Z" name: switch-graph-0a743.18c6bc894580bd7e namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 79053882-fa35-4c2e-a432-5feb1b462777 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T10:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0a743 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 24190e2b-33d7-4b88-a650-22cde8abe7a1 kind: Event lastTimestamp: "2026-07-29T10:40:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T10:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:40:15Z" name: switch-graph-0a743.18c6bc8945ee5fed namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: a40ccc53-d6b2-482a-aa53-589b5b22365a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T11:00:25.003182Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-965a2-5ddc4d9f8b-lpck6 to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T11:00:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T11:00:25Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6bda679b64f59 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: e000489d-42a0-4fba-a8da-482a11dd3994 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:00:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-965a2-serving-cert" not found' metadata: creationTimestamp: "2026-07-29T11:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:25Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6bda684bb8ed8 namespace: kserve-ci-e2e-test resourceVersion: "26757" uid: 53bc3a89-d225-403d-b894-83428c3446a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:26Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:00:26Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T11:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T11:00:26Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6bda6b776a948 namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: a4b8f816-9212-49f1-a49f-d570a4b17655 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-965a2} kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:00:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" already present on machine metadata: creationTimestamp: "2026-07-29T11:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:26Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6bda6b9151276 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 4b378eb3-7e9f-4346-811f-988827a5e064 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-965a2} kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:00:26Z" message: 'Created container: switch-graph-965a2' metadata: creationTimestamp: "2026-07-29T11:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:26Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6bda6bc4d937c namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: d69982ce-35cd-4132-bcb8-d294206d5318 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-965a2} kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:00:26Z" message: Started container switch-graph-965a2 metadata: creationTimestamp: "2026-07-29T11:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:00:26Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6bda6bcc66999 namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: bef4eae3-0c9b-4f90-be6a-ad6773a15ac1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-965a2} kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:16:44Z" message: Stopping container switch-graph-965a2 metadata: creationTimestamp: "2026-07-29T11:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:16:44Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6be8a80b5fdaa namespace: kserve-ci-e2e-test resourceVersion: "33603" uid: 6eae959f-7487-4da1-9eba-fe4874c95102 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T11:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-965a2} kind: Pod name: switch-graph-965a2-5ddc4d9f8b-lpck6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2fab34aa-33e0-436f-b2ba-c4bd93f9ee90 kind: Event lastTimestamp: "2026-07-29T11:17:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T11:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T11:17:12Z" name: switch-graph-965a2-5ddc4d9f8b-lpck6.18c6be8b42e191f2 namespace: kserve-ci-e2e-test resourceVersion: "33882" uid: 860c4556-5ac8-43fe-9b53-c18a3edb5ebd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-965a2-5ddc4d9f8b namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: 14b104fa-539c-439f-90d3-2fefd6955aeb kind: Event lastTimestamp: "2026-07-29T11:00:24Z" message: 'Created pod: switch-graph-965a2-5ddc4d9f8b-lpck6' metadata: creationTimestamp: "2026-07-29T11:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:24Z" name: switch-graph-965a2-5ddc4d9f8b.18c6bda67951de43 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: e89903b7-9774-4877-b7d3-03d9db751dc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-965a2 namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: 1bc864a9-1f2c-494b-81ea-f575e8806626 kind: Event lastTimestamp: "2026-07-29T11:00:24Z" message: Scaled up replica set switch-graph-965a2-5ddc4d9f8b from 0 to 1 metadata: creationTimestamp: "2026-07-29T11:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:24Z" name: switch-graph-965a2.18c6bda678719a9d namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: deafe2ab-170b-415a-b7d1-57ee6771227c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T11:00:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-965a2 namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: a6c06a61-3832-475a-ac40-7d6ac5fbc76b kind: Event lastTimestamp: "2026-07-29T11:00:25Z" message: InferenceGraph [switch-graph-965a2] is Ready metadata: creationTimestamp: "2026-07-29T11:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T11:00:25Z" name: switch-graph-965a2.18c6bda67d5ee7da namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 169e8c88-0ee6-4284-b294-b7336f90fd34 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T11:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-965a2 namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 8cf9a7f9-53bf-4c85-959a-464e2622044e kind: Event lastTimestamp: "2026-07-29T11:00:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T11:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:55Z" name: switch-graph-965a2.18c6bda9f6d75081 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: 142dfee8-fe14-4b66-9ffe-75f33c9fe065 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-29T11:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-965a2 namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 8cf9a7f9-53bf-4c85-959a-464e2622044e kind: Event lastTimestamp: "2026-07-29T11:00:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-29T11:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T11:00:55Z" name: switch-graph-965a2.18c6bda9f736b702 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 26946393-66ec-4b3f-a2d5-4206d6bc2fb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-29T10:38:34.625517Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f0182-68659b478c-prr6c to ip-10-0-129-138.ec2.internal metadata: creationTimestamp: "2026-07-29T10:38:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-29T10:38:34Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc75611de49d namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 2b70fc20-09fe-4b34-957d-020e4a476d53 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6888766bc5-npbb6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:38:35Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-29T10:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-29T10:38:35Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc757af68c35 namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: b624e73d-f38a-40d2-813b-1da313cd9381 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0182} kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:38:35Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" metadata: creationTimestamp: "2026-07-29T10:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:35Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc757c732f94 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 0ff4c7b9-fe81-4065-a794-2f2425113f73 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0182} kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:38:37Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:41cb8bf9ff6c7df05622085595fdb4c1e1d3a571d74d4bdea37a3b72ad46799b" in 2.312s (2.312s including waiting). Image size: 218226816 bytes.' metadata: creationTimestamp: "2026-07-29T10:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:37Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc76064abf95 namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: dcb10cf0-0996-444f-90c8-4ffd41a4cd4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0182} kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:38:37Z" message: 'Created container: switch-graph-f0182' metadata: creationTimestamp: "2026-07-29T10:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:37Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc7609ef94c7 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 33a736b0-358a-4b80-8049-dacaf76426b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0182} kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:38:37Z" message: Started container switch-graph-f0182 metadata: creationTimestamp: "2026-07-29T10:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:37Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc760a6c398f namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 4ca56c23-ac41-4699-8f0e-4b6ec6ad770b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0182} kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:38:44Z" message: Stopping container switch-graph-f0182 metadata: creationTimestamp: "2026-07-29T10:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:38:44Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc77bf404075 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: d6194725-e4c9-41b7-af2a-9c59f819d75b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-29T10:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0182} kind: Pod name: switch-graph-f0182-68659b478c-prr6c namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: bf0d8268-5080-4e2e-9058-949bfba19c7a kind: Event lastTimestamp: "2026-07-29T10:39:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-29T10:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-29T10:39:13Z" name: switch-graph-f0182-68659b478c-prr6c.18c6bc78a36b14e2 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 623b429b-ca18-41a7-b2d8-6635443cc180 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-138.ec2.internal source: component: kubelet host: ip-10-0-129-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f0182-68659b478c namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: bbbc0330-439e-46ab-aea2-63502c6a2b22 kind: Event lastTimestamp: "2026-07-29T10:38:34Z" message: 'Created pod: switch-graph-f0182-68659b478c-prr6c' metadata: creationTimestamp: "2026-07-29T10:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:34Z" name: switch-graph-f0182-68659b478c.18c6bc7560a422c8 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: a9a78e7b-0408-42f2-88ba-bab7b3e81325 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f0182 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 7017684b-3246-4ed3-993f-e1c0bd2a250f kind: Event lastTimestamp: "2026-07-29T10:38:34Z" message: Scaled up replica set switch-graph-f0182-68659b478c from 0 to 1 metadata: creationTimestamp: "2026-07-29T10:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-29T10:38:34Z" name: switch-graph-f0182.18c6bc755fd80374 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 06a6dd20-8284-4627-af8a-bda534f9287a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f0182 namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 09ee8e12-6eec-4982-84ab-3bcdc9619924 kind: Event lastTimestamp: "2026-07-29T10:38:34Z" message: InferenceGraph [switch-graph-f0182] is Ready metadata: creationTimestamp: "2026-07-29T10:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:34Z" name: switch-graph-f0182.18c6bc7564dd85d9 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 1a15cac4-15ec-4618-82b1-c45b7bbb0a55 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f0182 namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 09ee8e12-6eec-4982-84ab-3bcdc9619924 kind: Event lastTimestamp: "2026-07-29T10:38:34Z" message: 'Failed to update status for InferenceGraph "switch-graph-f0182": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-f0182": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:34Z" name: switch-graph-f0182.18c6bc75671b119c namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: f2df7351-9830-469b-9d9a-0af90c7b80b5 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-29T10:38:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f0182 namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 09ee8e12-6eec-4982-84ab-3bcdc9619924 kind: Event lastTimestamp: "2026-07-29T10:38:34Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-f0182": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-29T10:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-29T10:38:34Z" name: switch-graph-f0182.18c6bc75671b6340 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 1897fa9e-ee36-4cba-a522-8bbe05f80782 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "33884"