--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:01.873439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1da67-9557f86c-ptfbc to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:01Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d72245caf7 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: dd16bf23-10e5-4848-8d23-3bb94394f472 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:02Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: "2026-07-16T17:57:02Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:02Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d73c17f85b namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: def35793-02cb-4094-9728-2f07ddf6d5f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1da67} kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: "2026-07-16T17:57:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:02Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d73d687bd5 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 3be098d7-88cd-41b9-a0de-9055b6d2fdaa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1da67} kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: "2026-07-16T17:57:02Z" message: 'Created container: ensemble-graph-1da67' metadata: creationTimestamp: "2026-07-16T17:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:02Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d7407b620e namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 5a32cf5e-931e-4869-aab2-20c5b79015db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1da67} kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: "2026-07-16T17:57:02Z" message: Started container ensemble-graph-1da67 metadata: creationTimestamp: "2026-07-16T17:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:02Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d740ed1140 namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 66ab9390-865c-4bf9-8172-b8d7396cbf7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1da67} kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Stopping container ensemble-graph-1da67 metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d98129eba8 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: cbb3f26a-6cc4-4b14-8a54-af7fe32cbb53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1da67} kind: Pod name: ensemble-graph-1da67-9557f86c-ptfbc namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c5706312-1686-4264-b434-6d87298148b3 kind: Event lastTimestamp: "2026-07-16T17:57:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:38Z" name: ensemble-graph-1da67-9557f86c-ptfbc.18c2d6d9e81afde9 namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: bbe83362-28f0-4184-bf5e-f8f6a54d9847 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1da67-9557f86c namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: d498877d-65a8-4129-afe1-215d8596148d kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: 'Created pod: ensemble-graph-1da67-9557f86c-ptfbc' metadata: creationTimestamp: "2026-07-16T17:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:01Z" name: ensemble-graph-1da67-9557f86c.18c2d6d7215276c6 namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: 29b35646-331b-4ef7-b898-c555c5b7377e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1da67 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: a44e43a4-093f-42fd-a3ad-128037e3d33a kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: Scaled up replica set ensemble-graph-1da67-9557f86c from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:01Z" name: ensemble-graph-1da67.18c2d6d7203244a7 namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: e76e1b93-e812-42e6-9f81-b5ef59d25a9b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1da67 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 62d5f057-5229-4771-8446-776849fe35cc kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: InferenceGraph [ensemble-graph-1da67] is Ready metadata: creationTimestamp: "2026-07-16T17:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:01Z" name: ensemble-graph-1da67.18c2d6d7285b0d34 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 906847a4-b190-4e48-a4d8-a4a4ccee7b19 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:58:12.340441Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-efe2d-c6c68b6dc-xnx4n to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:58:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:58:12Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d6e78a6ed276 namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 30379c9a-015a-46d1-a862-c5e443a42d2f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:12Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T17:58:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-efe2d-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:12Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d6e796e44e67 namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 4c69f5f3-bb7b-4bf6-9f1a-c230cd044633 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T17:58:13Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:58:13Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d6e7c7889a7a namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: 1f456a88-ec78-4a93-8ea6-1530603d0b4b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-efe2d} kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T17:58:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:13Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d6e7c8dcdce7 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: e1074bc0-6a7d-4123-ad56-298bc12081e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-efe2d} kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T17:58:13Z" message: 'Created container: ensemble-graph-efe2d' metadata: creationTimestamp: "2026-07-16T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:13Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d6e7cbdd54a0 namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 6aaec3cc-016f-4ad7-9838-7351c02ea99d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-efe2d} kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T17:58:13Z" message: Started container ensemble-graph-efe2d metadata: creationTimestamp: "2026-07-16T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:13Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d6e7cc4b66a2 namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: 3c55a79b-5f34-4f0a-bd57-e1f06ca096b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-efe2d} kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T18:06:26Z" message: Stopping container ensemble-graph-efe2d metadata: creationTimestamp: "2026-07-16T18:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:26Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d75ab4ffac6d namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: a4829e71-953b-4bde-8266-d75b05d1cf70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-efe2d} kind: Pod name: ensemble-graph-efe2d-c6c68b6dc-xnx4n namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 61f7e3d7-08fa-422a-ba60-5dbf75eff86c kind: Event lastTimestamp: "2026-07-16T18:06:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T18:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:55Z" name: ensemble-graph-efe2d-c6c68b6dc-xnx4n.18c2d75b7087c544 namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: 58aa0b8d-530e-4281-8a1b-9a44f7ea2f5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-efe2d-c6c68b6dc namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 27a44bcf-b648-4395-886e-af0ec2524260 kind: Event lastTimestamp: "2026-07-16T17:58:12Z" message: 'Created pod: ensemble-graph-efe2d-c6c68b6dc-xnx4n' metadata: creationTimestamp: "2026-07-16T17:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:12Z" name: ensemble-graph-efe2d-c6c68b6dc.18c2d6e789eda10c namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: 9d23bf81-59a3-4c49-8809-1f4fd7d1dddd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-efe2d namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: eea9a79d-80cb-4a01-857b-4a9d25aec6d0 kind: Event lastTimestamp: "2026-07-16T17:58:12Z" message: Scaled up replica set ensemble-graph-efe2d-c6c68b6dc from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:12Z" name: ensemble-graph-efe2d.18c2d6e788d65f1c namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: 2d45dd16-3715-4bf9-b25f-aa3af570408a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-efe2d namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 6fd0b272-0855-4ccf-8c53-4f4d92f688ad kind: Event lastTimestamp: "2026-07-16T17:58:12Z" message: InferenceGraph [ensemble-graph-efe2d] is Ready metadata: creationTimestamp: "2026-07-16T17:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:12Z" name: ensemble-graph-efe2d.18c2d6e78fbf62fa namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: bbb84e26-563e-4afb-9ff3-e94a6576f473 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-efe2d namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 16d4fb5d-e0a2-486a-8b97-c34ef18f0238 kind: Event lastTimestamp: "2026-07-16T17:58:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:42Z" name: ensemble-graph-efe2d.18c2d6eb07521515 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: ebb96a74-fd46-4073-8f62-f848a8cda2b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-efe2d namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 16d4fb5d-e0a2-486a-8b97-c34ef18f0238 kind: Event lastTimestamp: "2026-07-16T17:58:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:42Z" name: ensemble-graph-efe2d.18c2d6eb079fdb00 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: 11322808-43fc-4dd6-8009-e7d784f5d641 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:56:01.963979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1da67-predictor-5c69b7464d-wgnzr to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:56:01Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c92f6400e8 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 820f429e-c212-4895-a275-0185ffc29bf5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1da67-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:02Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c939db98c2 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 3cc48ab4-61c8-4977-b346-3a2800e184f6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c96d38aea0 namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 37fb37b9-ee26-43c8-b21c-7ec5f3acc4b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c96f1de3ab namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: 52d40266-b32f-4b2d-9a0f-51e90686f5e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c97227b3da namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: 6231c94f-1fde-4aa9-a52b-8dec50f17c4f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c972a87b15 namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: cd6eedfc-7ddb-42b8-abfe-3a65b0aa6040 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c972b1fd19 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: e00e8cfb-dbbc-4fce-8374-8a7cc0763c81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c97559152a namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 9d38245c-ce99-4bc3-b6af-64aed80c8df9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c975ca6c63 namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: b9f456a9-3e8a-480f-bbc4-a62fe286bb5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:56:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:39Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6c988000c3a namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 6996f34f-89db-4f4c-8c86-d1bfe8543aa5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6d98b47b978 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 2704e2aa-dd28-4031-b725-6f69bb12a6a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6d98b4847a4 namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: fd9a090e-e574-458e-9c3a-612f1d2cbb4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b1e18acf-38af-4f3e-87db-3375d62803a9 kind: Event lastTimestamp: "2026-07-16T17:57:14Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:14Z" name: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr.18c2d6da103cf0b1 namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: e4c879bb-3294-4fbc-a859-c5bc02416afa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1da67-predictor-5c69b7464d namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: ba560b9c-5e56-4296-bf10-d57bf2d51903 kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: 'Created pod: error-404-isvc-1da67-predictor-5c69b7464d-wgnzr' metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:01Z" name: error-404-isvc-1da67-predictor-5c69b7464d.18c2d6c92efbdd71 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 7546b427-893a-495c-a362-42296badbfd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 3512e2f5-3acc-49ad-8e33-f3b73e0b16ea kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Scaled up replica set error-404-isvc-1da67-predictor-5c69b7464d from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:01Z" name: error-404-isvc-1da67-predictor.18c2d6c92d2ebe58 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: fb11927f-1bb9-40ea-a9a0-3745de954104 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: bdac7b7f-9661-4192-999e-c44d4675bf43 kind: Event lastTimestamp: "2026-07-16T17:56:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:31Z" name: error-404-isvc-1da67-predictor.18c2d6ccabaaad00 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: 387663e7-dbe2-433e-9f24-78bb1ea2106f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: bdac7b7f-9661-4192-999e-c44d4675bf43 kind: Event lastTimestamp: "2026-07-16T17:56:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:31Z" name: error-404-isvc-1da67-predictor.18c2d6ccac4ffac1 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 92d59f64-2668-466c-a020-fb56042e8fe8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: bdac7b7f-9661-4192-999e-c44d4675bf43 kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:02Z" name: error-404-isvc-1da67-predictor.18c2d6d3aa0f7dab namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 494e838d-7e80-4294-b4d3-24126e1b7c24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: bdac7b7f-9661-4192-999e-c44d4675bf43 kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:02Z" name: error-404-isvc-1da67-predictor.18c2d6d3aa89def0 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 61301560-205e-44aa-9beb-6ebcfe770998 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1da67 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: e7bb2f83-f719-499a-b5e2-14c2f1511a38 kind: Event lastTimestamp: "2026-07-16T17:56:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:49Z" name: error-404-isvc-1da67.18c2d6c92d1e1da8 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 56cabb4a-219d-404c-95b7-8717462686c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1da67 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: e7bb2f83-f719-499a-b5e2-14c2f1511a38 kind: Event lastTimestamp: "2026-07-16T17:56:49Z" message: InferenceService [error-404-isvc-1da67] is Ready metadata: creationTimestamp: "2026-07-16T17:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:49Z" name: error-404-isvc-1da67.18c2d6d443ca0627 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 6e019f8f-fe75-4b3b-b632-5282a56af53a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:07:37.629927Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b284fe39f namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 72f872f3-a080-4809-a4cb-6ffc5efdb779 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b41c75eb0 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: a2ed16fa-b441-4ee5-b80d-449a276f3f1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b4340ba58 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: 1d66baec-d183-43d0-8fe0-9e4c5539c9fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b4629837f namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: 2dcd1e29-e43b-4c2f-86a8-77bb1858bd78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b469c4aa8 namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: 50e04453-44d4-4c09-9d29-d545dd0db191 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b46a539bb namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: ad6adcaa-209d-454e-96d2-03825ae621b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b4906981f namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 1ca53088-461e-4fc7-9389-910423c8fdfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b498bd6bc namespace: kserve-ci-e2e-test resourceVersion: "26599" uid: 6e9a16a7-a667-49be-9fe2-ac016207caa2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:16:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:54Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d76b4b248715 namespace: kserve-ci-e2e-test resourceVersion: "30695" uid: 8e65e7a9-a83a-4f09-89b9-60599e34571f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:16:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:52Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d7ec54e5c702 namespace: kserve-ci-e2e-test resourceVersion: "30676" uid: efedaea5-cc39-4509-9863-a9d4491ea6c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:16:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:52Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d7ec54e65524 namespace: kserve-ci-e2e-test resourceVersion: "30677" uid: b916f18a-e37e-4d8e-95c3-e53d42739ee2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 2817fb1c-1f8c-4eda-86be-79a4078085ab kind: Event lastTimestamp: "2026-07-16T18:16:54Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:54Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7.18c2d7ecbfd2839e namespace: kserve-ci-e2e-test resourceVersion: "30694" uid: 43a15f39-d792-4b34-8c2f-2e0b8cc5f7bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-21cc5-predictor-5d5c5f9fdb namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 94f651ee-b186-4b7f-a821-b15a7ef7522f kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'Created pod: error-404-isvc-21cc5-predictor-5d5c5f9fdb-xnxd7' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-21cc5-predictor-5d5c5f9fdb.18c2d76b27da9a25 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 2fdad3cd-d36e-4896-9e84-a14b879dfc35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: ecae13f7-ca27-449a-b1fd-e0a063b5979e kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Scaled up replica set error-404-isvc-21cc5-predictor-5d5c5f9fdb from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-21cc5-predictor.18c2d76b2662d8c4 namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 4900941a-c762-400a-805c-e2a10a200ea6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 2dd8beb5-8163-4eae-90a3-ab6c50f1f0be kind: Event lastTimestamp: "2026-07-16T18:07:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:52Z" name: error-404-isvc-21cc5-predictor.18c2d76ea51b7608 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 548c5508-e2c8-456e-bd68-0c36d7564e8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 2dd8beb5-8163-4eae-90a3-ab6c50f1f0be kind: Event lastTimestamp: "2026-07-16T18:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:52Z" name: error-404-isvc-21cc5-predictor.18c2d76ea58e8014 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: e6cba0d7-5a42-47fe-bdea-c596cc69f08e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T18:08:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2dd8beb5-8163-4eae-90a3-ab6c50f1f0be kind: Event lastTimestamp: "2026-07-16T18:09:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:09:07Z" name: error-404-isvc-21cc5-predictor.18c2d772245316e9 namespace: kserve-ci-e2e-test resourceVersion: "27420" uid: 354088be-63e4-46c5-bd0f-bda409efdce2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T18:08:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2dd8beb5-8163-4eae-90a3-ab6c50f1f0be kind: Event lastTimestamp: "2026-07-16T18:09:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:09:07Z" name: error-404-isvc-21cc5-predictor.18c2d77224c703d6 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 8173e1ad-4798-4149-a8a6-3cd2cde3740d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: ac2b6b6b-c05d-4cf2-bf29-9843cffd1eff kind: Event lastTimestamp: "2026-07-16T18:08:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:08:24Z" name: error-404-isvc-21cc5.18c2d76b2655e8ff namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 45084bea-6dbe-43f0-a9a1-8913447c7c48 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: ac2b6b6b-c05d-4cf2-bf29-9843cffd1eff kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-21cc5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-21cc5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-21cc5.18c2d76b26efbc6e namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: a681de7d-72aa-473c-a34d-d86d50dbb45c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: ac2b6b6b-c05d-4cf2-bf29-9843cffd1eff kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-21cc5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-21cc5.18c2d76b26effaed namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 872d9e01-66a6-4404-b479-29b430faddb4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: ac2b6b6b-c05d-4cf2-bf29-9843cffd1eff kind: Event lastTimestamp: "2026-07-16T18:08:24Z" message: InferenceService [error-404-isvc-21cc5] is Ready metadata: creationTimestamp: "2026-07-16T18:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:08:24Z" name: error-404-isvc-21cc5.18c2d77606840598 namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: 94aebc80-c99e-45ca-b032-fddf95ee667a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:47:26.933204Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-38e88-predictor-859fd4f7c8-bfsds to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:47:26Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d651452a0219 namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 0f7a35eb-92c5-40dc-9aa0-a5bb8f9c252d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d6515ec5afaf namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 1c424743-1334-4d63-ae63-eb077ae1a9bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d651603fb823 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 91ea8da1-55ae-4129-883f-8e9535f5c9d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d6516306a8cf namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 8c0ef148-8fa9-4979-b53e-12e65f8e2344 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d65163704e9c namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 65e28a03-c027-488d-9ee5-6cbc38d7ff51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d6516379fb82 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 830ed33a-4c81-4943-a1ab-bc7f6052a7d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d65166079d94 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 1c341704-e944-4720-b70b-d5cc407a54fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d651666feff7 namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: c778cda5-0333-4035-bd11-e44f4c6362a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:48:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:05Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d651dcf8aa50 namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 9e75ee0b-01b4-408c-bea0-4cafe2174afc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d6d266da5c25 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 4e6e7d29-202f-49b5-9d61-b9297a6d0fff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 32b3e31a-3cea-433f-84c1-6d4da383701b kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds.18c2d6d266dacec5 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: d9da71b1-92da-4da0-9c9c-e17d526558af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-38e88-predictor-859fd4f7c8 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: cc060a5d-22c9-4724-a177-8f607b1e2956 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: 'Created pod: error-404-isvc-38e88-predictor-859fd4f7c8-bfsds' metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:26Z" name: error-404-isvc-38e88-predictor-859fd4f7c8.18c2d65144d1c3f8 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: ff780bd4-859c-4f75-bf48-554dd263ac7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: c98cc54a-e1bc-4dfc-b66d-eac59c5f7a99 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Scaled up replica set error-404-isvc-38e88-predictor-859fd4f7c8 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:26Z" name: error-404-isvc-38e88-predictor.18c2d65142e05f2c namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: b1bcbea7-8a4c-4a36-9ec6-392160f1be57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 03849085-08da-4243-b0ba-4b1c8395fcc7 kind: Event lastTimestamp: "2026-07-16T17:47:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:56Z" name: error-404-isvc-38e88-predictor.18c2d654c160dc41 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 2d386cd9-18e8-4beb-82fb-cbe3e4c0059c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 03849085-08da-4243-b0ba-4b1c8395fcc7 kind: Event lastTimestamp: "2026-07-16T17:47:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:56Z" name: error-404-isvc-38e88-predictor.18c2d654c1d69d00 namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: a4c2ad7c-cec4-4e92-996a-22ce77d85b6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 03849085-08da-4243-b0ba-4b1c8395fcc7 kind: Event lastTimestamp: "2026-07-16T17:48:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:41Z" name: error-404-isvc-38e88-predictor.18c2d65bbf4b85ad namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: dbe50056-fe3a-4ebf-a6ec-7d462d8897c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 03849085-08da-4243-b0ba-4b1c8395fcc7 kind: Event lastTimestamp: "2026-07-16T17:48:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:41Z" name: error-404-isvc-38e88-predictor.18c2d65bbfcfd6f5 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 37a098c9-e044-49f3-b9e3-4e48048c4f0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: ccfe8f30-895f-4ed7-8518-6ff82cce84a5 kind: Event lastTimestamp: "2026-07-16T17:48:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:15Z" name: error-404-isvc-38e88.18c2d6514306767a namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: bc8296d6-baa8-42fb-8027-cd23447b3947 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: ccfe8f30-895f-4ed7-8518-6ff82cce84a5 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-38e88": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-38e88": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:26Z" name: error-404-isvc-38e88.18c2d6514369c780 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 4d6ab0b7-41a7-48cf-a304-72a1763aa1e3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: ccfe8f30-895f-4ed7-8518-6ff82cce84a5 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-38e88": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:26Z" name: error-404-isvc-38e88.18c2d651436a15db namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: c70862b2-e4fc-418a-80a8-238da8dc66d1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: ccfe8f30-895f-4ed7-8518-6ff82cce84a5 kind: Event lastTimestamp: "2026-07-16T17:48:15Z" message: InferenceService [error-404-isvc-38e88] is Ready metadata: creationTimestamp: "2026-07-16T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:15Z" name: error-404-isvc-38e88.18c2d65c999a3965 namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: e397fe54-0bd3-40d0-8c81-ca09880cc353 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:56:41.767312Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6687f-predictor-547df57596-fmjkv to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d273da9bfa namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: 364bef74-1e19-40fb-ad2f-022a839ec431 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d28e393ceb namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 8fb5f539-ee3b-4003-9239-e881b6363c0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d28f9e0e2c namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 0cdb744a-ff92-42d4-9ab6-be9120837eb9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d291bd8cf4 namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: d4287ab9-c500-4655-94fb-d33e8c747bcb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d2922d2ba6 namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 13873128-4097-4af9-bff1-4525c5abd42d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d29235680b namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: c78cd88b-fbc0-45d4-83fe-2f8a476f0358 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d2957abdec namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 568ac4fa-2d84-47bb-a579-089469f57f17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d295ea05b9 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 6b6732a2-7af9-41f0-b993-8607cf300108 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:57:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:18Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6d2a64ecf0b namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: a24c7ff4-7b4d-4941-97c6-8772bb0fc74c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:51Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6e2c513c2b0 namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 1d778b82-c68f-4278-b077-c347488fad5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:51Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6e2c514c6b0 namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 82346cc8-9230-42ae-a34f-a46a35738270 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6687f-predictor-547df57596-fmjkv namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 822c7304-5d93-443d-aa03-1d20f646a951 kind: Event lastTimestamp: "2026-07-16T17:57:53Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:53Z" name: error-404-isvc-6687f-predictor-547df57596-fmjkv.18c2d6e32e9b80c2 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 8f6bca80-dd6b-451b-86b6-150545ff9fb8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6687f-predictor-547df57596 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: ce6bb755-2c9e-4523-b058-9246a0fca75e kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: 'Created pod: error-404-isvc-6687f-predictor-547df57596-fmjkv' metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-6687f-predictor-547df57596.18c2d6d273840789 namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 948ff552-550e-46e7-ab52-36cca949b757 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: fa5c4652-8274-4d1b-a717-2e608e806f0d kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Scaled up replica set error-404-isvc-6687f-predictor-547df57596 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-6687f-predictor.18c2d6d2717751d2 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: 800197d2-e55e-4d61-be31-3babf4a0837e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: cc649c16-18c3-470a-993e-0c08d2235537 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:41Z" name: error-404-isvc-6687f-predictor.18c2d6d5efd44401 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: ffa76f84-dbd3-4b0d-9327-237180db411a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: cc649c16-18c3-470a-993e-0c08d2235537 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:41Z" name: error-404-isvc-6687f-predictor.18c2d6d5f025b4e5 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 17fe3fe6-6b29-4668-a21a-a0b79f79fb30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f1b33b6e-1c86-4190-86f8-d5c6863edaf6 kind: Event lastTimestamp: "2026-07-16T17:57:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:28Z" name: error-404-isvc-6687f.18c2d6d27110b40f namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 9739b662-6854-4710-ab4d-aa0e788ca84f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f1b33b6e-1c86-4190-86f8-d5c6863edaf6 kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-6687f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6687f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-6687f.18c2d6d2719c40c4 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: dd533694-358d-4c82-9874-0b37a43e1da1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f1b33b6e-1c86-4190-86f8-d5c6863edaf6 kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6687f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:41Z" name: error-404-isvc-6687f.18c2d6d2719c886a namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 78606ca3-1879-4637-be2f-bf5c6cb1f345 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: f1b33b6e-1c86-4190-86f8-d5c6863edaf6 kind: Event lastTimestamp: "2026-07-16T17:57:28Z" message: InferenceService [error-404-isvc-6687f] is Ready metadata: creationTimestamp: "2026-07-16T17:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:28Z" name: error-404-isvc-6687f.18c2d6dd61e4f4db namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 29441406-92af-49d0-879d-f11a3c3f8425 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:07:06.866666Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-97269-predictor-7d7f6f997b-9t55g to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d763feadbb74 namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: 22aedce3-7bf3-4adb-a344-725481bfa0c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-97269-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d764068c6c14 namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: ac6951b1-d8a8-4cc6-bf1a-c236ca1cc435 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:07:07Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d7643c238d51 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 31ba649b-fef9-498e-9ae7-8a77d1af28d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d7643d717377 namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: 3bb7158d-7bf5-4cb5-a882-5d5b1fe4d3cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d7644097fa60 namespace: kserve-ci-e2e-test resourceVersion: "26148" uid: 68710306-d022-4e44-a34f-62987ed00c51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d764417f852e namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: 03a15a01-b420-4183-845a-442ebb74aa95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d7644189c09c namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: bd6c6ded-e1a1-4994-8210-6b91c6aec461 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:08Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d7644451c0f0 namespace: kserve-ci-e2e-test resourceVersion: "26151" uid: c881d5b0-da64-4a6b-ad20-62f3881067d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:08Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d76444c80329 namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 8ce8ba36-e510-44c5-9949-d97b522749d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:07:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:44Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d76446e56dc3 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: b9ebe9aa-5183-40c1-8225-089edb86f4aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:24:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:24:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:24:26Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d855f79427aa namespace: kserve-ci-e2e-test resourceVersion: "33815" uid: 08303878-1ad3-403e-96c6-bbb6ff56539a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:24:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: ae46f104-584c-44ce-ae08-507d364f8971 kind: Event lastTimestamp: "2026-07-16T18:24:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:24:26Z" name: error-404-isvc-97269-predictor-7d7f6f997b-9t55g.18c2d855f794d11b namespace: kserve-ci-e2e-test resourceVersion: "33816" uid: b04ae4ca-eed2-4e96-8ac5-015e74431d81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-97269-predictor-7d7f6f997b namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 2d71e1c2-0359-41a9-9fa2-95f266b8e79f kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'Created pod: error-404-isvc-97269-predictor-7d7f6f997b-9t55g' metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-97269-predictor-7d7f6f997b.18c2d763fe6c7132 namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: e86ea4b7-6d4d-4179-987e-227e53f9b03b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26095" uid: ce0641e5-4f27-46e9-ab77-38df8ed7084a kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Scaled up replica set error-404-isvc-97269-predictor-7d7f6f997b from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-97269-predictor.18c2d763fc9b9abe namespace: kserve-ci-e2e-test resourceVersion: "26100" uid: c7252b2f-384a-41e5-91e2-19f6414a798b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: 1a71b44f-7c72-42c5-a5ea-42b42f749c82 kind: Event lastTimestamp: "2026-07-16T18:07:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:21Z" name: error-404-isvc-97269-predictor.18c2d7677afbe824 namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 2277abf4-ed58-44be-b5e4-436164195a26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: 1a71b44f-7c72-42c5-a5ea-42b42f749c82 kind: Event lastTimestamp: "2026-07-16T18:07:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:21Z" name: error-404-isvc-97269-predictor.18c2d7677b75d956 namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 27801807-1aa2-4074-9631-dd7a2c5cee0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T18:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: 1a71b44f-7c72-42c5-a5ea-42b42f749c82 kind: Event lastTimestamp: "2026-07-16T18:08:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:21Z" name: error-404-isvc-97269-predictor.18c2d76afa669c19 namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: d7c55936-b8a1-4437-81a9-0a24aea0fe72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T18:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: 1a71b44f-7c72-42c5-a5ea-42b42f749c82 kind: Event lastTimestamp: "2026-07-16T18:08:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:21Z" name: error-404-isvc-97269-predictor.18c2d76afae46d14 namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: d0b4789b-90bd-41cb-bb5f-a1e4f2876e50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 041093cf-ed47-4312-8aad-01cfdda084d6 kind: Event lastTimestamp: "2026-07-16T18:07:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:54Z" name: error-404-isvc-97269.18c2d763fc7beebd namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: cb0f9277-464d-4c14-b02c-3f64f35dc854 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 041093cf-ed47-4312-8aad-01cfdda084d6 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-97269": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-97269": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-97269.18c2d763fcc7fe04 namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 6387f36f-984e-43e2-b530-85f767a4e220 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 041093cf-ed47-4312-8aad-01cfdda084d6 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-97269": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-97269.18c2d763fcc84002 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: 2c2769a4-f89d-4ff5-9851-bb013bca4c35 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 041093cf-ed47-4312-8aad-01cfdda084d6 kind: Event lastTimestamp: "2026-07-16T18:07:54Z" message: InferenceService [error-404-isvc-97269] is Ready metadata: creationTimestamp: "2026-07-16T18:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:54Z" name: error-404-isvc-97269.18c2d76f023722de namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: b7dcbd22-963c-41b1-9384-7a120f32b5f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:06:27.310255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a019b-predictor-5578757556-bfcld to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75ac8eed5f1 namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: fed6653d-c0d7-4e24-87a6-26bffa638a1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a019b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75ad343bf80 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: ea5e9470-4bd2-4694-a1c7-318e43587e13 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b061ded30 namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: 5ae34ed3-f101-4d28-9120-30c8f56bdfd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b076fa3de namespace: kserve-ci-e2e-test resourceVersion: "25690" uid: a68501a6-2349-4e39-98ed-1e2c2130cdee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b0a7fd37c namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 84b6f06c-032b-469c-81f0-3537733a38ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b0af3facd namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: f6911ea3-cc50-4bfa-a8ac-55b7a932495b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b0afc19c8 namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 9e43aa16-1434-4894-ba58-ff3f62aaeb1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b0dbcc964 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: e8690fbe-9d6c-4ebe-a395-afe771656284 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:06:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:28Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b0e2e9f22 namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: fcfd684d-ddce-4028-ab3b-096de5485322 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:07:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:05Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d75b644bedd3 namespace: kserve-ci-e2e-test resourceVersion: "26001" uid: c64e1ef2-7a4f-47b0-8e01-c00a963ee257 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d76b1bdc54bc namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: e5edca45-7b1d-4103-9ac3-0ffe7f48afa9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:37Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d76b1bdcec1d namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 9e88b214-aaab-40e8-8317-c05b10f2fcfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a019b-predictor-5578757556-bfcld namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d8f53a82-a7e5-4703-9241-d93ff28b09c6 kind: Event lastTimestamp: "2026-07-16T18:07:40Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:40Z" name: error-404-isvc-a019b-predictor-5578757556-bfcld.18c2d76bec8682b9 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 7626f025-e4f1-4037-b12e-25f368c1d2cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a019b-predictor-5578757556 namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: e3b3eff7-a3c5-40d6-9688-0667859819fc kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: 'Created pod: error-404-isvc-a019b-predictor-5578757556-bfcld' metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-a019b-predictor-5578757556.18c2d75ac8873655 namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 2258a25b-4160-4d71-aabf-ee66696d8ca6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 888f46e0-f495-4b6e-9678-1f6c8bcae52d kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: Scaled up replica set error-404-isvc-a019b-predictor-5578757556 from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-a019b-predictor.18c2d75ac51db368 namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 8c3d70ae-43aa-40f8-83c3-ad0023d62115 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 434dcca7-fc27-4e90-86ea-c1114ed5338c kind: Event lastTimestamp: "2026-07-16T18:06:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:06:57Z" name: error-404-isvc-a019b-predictor.18c2d75e439b308c namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: 6c9091d1-e509-458d-b7d3-98f302e14535 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 434dcca7-fc27-4e90-86ea-c1114ed5338c kind: Event lastTimestamp: "2026-07-16T18:06:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:06:57Z" name: error-404-isvc-a019b-predictor.18c2d75e440e10f3 namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: 6ac9f7cc-637e-4c15-bbbe-8efa65622b71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:07:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 434dcca7-fc27-4e90-86ea-c1114ed5338c kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:27Z" name: error-404-isvc-a019b-predictor.18c2d76541dd9368 namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: ba3e19f2-89e6-4da8-93f6-546c23aad6c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:07:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 434dcca7-fc27-4e90-86ea-c1114ed5338c kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:27Z" name: error-404-isvc-a019b-predictor.18c2d765424c3b33 namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: 96756c4e-6edd-4e38-b697-923e535ff31c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a019b namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a40d26f6-9d6c-436e-bacd-a006497de01f kind: Event lastTimestamp: "2026-07-16T18:07:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:16Z" name: error-404-isvc-a019b.18c2d75ac4efbf2a namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: a2841770-8cb8-40a4-b163-7ccaf02e4a5e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a019b namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a40d26f6-9d6c-436e-bacd-a006497de01f kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-a019b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a019b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-a019b.18c2d75ac59fb4c0 namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: 2bce4021-ea9f-4e0b-84bf-b03b5386a538 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a019b namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a40d26f6-9d6c-436e-bacd-a006497de01f kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a019b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-a019b.18c2d75ac5a00f6c namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 658ffbba-e802-44f6-b5e0-9267ea293966 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a019b namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: a40d26f6-9d6c-436e-bacd-a006497de01f kind: Event lastTimestamp: "2026-07-16T18:07:16Z" message: InferenceService [error-404-isvc-a019b] is Ready metadata: creationTimestamp: "2026-07-16T18:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:16Z" name: error-404-isvc-a019b.18c2d7661f8d33d9 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 17615459-57b0-4c6b-b3cc-c749c76e319f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:52.052035Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2d126608f namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 8be2bda2-7b0e-4c41-b467-33d9adf8c4cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2eb742de1 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: 2708bfd9-8305-416f-a765-c3e9417ddcbb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2ecbcf202 namespace: kserve-ci-e2e-test resourceVersion: "21564" uid: 967e2f60-f74e-4c1e-94db-3f4a0dcf0be8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2ef4e0eb9 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: 1608d072-39f1-448a-ab3e-f54cd88b855e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2efbd62d8 namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 0d07d2a2-7a64-440e-9831-2c18601fd58a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2efc7723b namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 5125065c-3c04-4a49-b860-913879ce0811 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2f2a2cf4a namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 8a273cc2-4b5b-4c6a-bf3f-e54172f08260 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e2f317c14a namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: c9278935-d20d-4460-818e-10b6f2155cfd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T18:07:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:08Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d6e3057c842f namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: d4f803ff-9c34-43f1-b420-14ed0bd0ce43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d763f1e45c98 namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: d970b3ca-34a0-4550-98bc-a585cb109a38 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d763f1e50792 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 1e2ccf3c-7307-4e09-b79a-bd2531de675c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: aad71ae1-a1c9-40ad-9fc8-5f0733be67bd kind: Event lastTimestamp: "2026-07-16T18:07:08Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:08Z" name: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb.18c2d76479e19423 namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: 673dfb91-8a8c-4950-ae16-120689155959 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a90bb-predictor-55f9ccb694 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 011883e5-56b4-4290-84bb-843d13537455 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'Created pod: error-404-isvc-a90bb-predictor-55f9ccb694-4vvcb' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor-55f9ccb694.18c2d6e2d0d6e59e namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 5c61924c-9be6-45d0-80be-8e407e737fb7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: dd2b0fca-a7b8-4ee7-a63b-423ff7d1c3f4 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Scaled up replica set error-404-isvc-a90bb-predictor-55f9ccb694 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb-predictor.18c2d6e2ceeac771 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 492c1adc-271b-48b1-82f8-c7b82c2554cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 5423af9d-e4a5-4481-ac43-03d157a2f3a0 kind: Event lastTimestamp: "2026-07-16T17:58:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:22Z" name: error-404-isvc-a90bb-predictor.18c2d6e64d5bf1d9 namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: b676da3a-8b34-40af-8ca6-2111082cb586 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 5423af9d-e4a5-4481-ac43-03d157a2f3a0 kind: Event lastTimestamp: "2026-07-16T17:58:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:22Z" name: error-404-isvc-a90bb-predictor.18c2d6e64dccce1b namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 1a439048-7e62-4424-818f-6c6982ef688a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:58:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 5423af9d-e4a5-4481-ac43-03d157a2f3a0 kind: Event lastTimestamp: "2026-07-16T17:59:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:07Z" name: error-404-isvc-a90bb-predictor.18c2d6ed4bc7c4e7 namespace: kserve-ci-e2e-test resourceVersion: "22275" uid: dab9a9a4-8ba2-422e-a718-75c32265e04b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:58:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 5423af9d-e4a5-4481-ac43-03d157a2f3a0 kind: Event lastTimestamp: "2026-07-16T17:59:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:07Z" name: error-404-isvc-a90bb-predictor.18c2d6ed4c46a696 namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: cc57138a-7ef1-48c8-bc37-d0e696522f2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: bae42f58-19b0-4960-978e-2df44af58c21 kind: Event lastTimestamp: "2026-07-16T17:58:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:39Z" name: error-404-isvc-a90bb.18c2d6e2ced658a0 namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: d5498b47-31fd-4bdf-867e-d822708e8775 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: bae42f58-19b0-4960-978e-2df44af58c21 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-a90bb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a90bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb.18c2d6e2cf374aad namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 8cf54331-6de0-413d-b7f4-05ec125cc20c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: bae42f58-19b0-4960-978e-2df44af58c21 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a90bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:52Z" name: error-404-isvc-a90bb.18c2d6e2cf379e2a namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 8ff7b16f-2e1b-468a-87eb-65d9fbac0a6c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: bae42f58-19b0-4960-978e-2df44af58c21 kind: Event lastTimestamp: "2026-07-16T17:58:39Z" message: InferenceService [error-404-isvc-a90bb] is Ready metadata: creationTimestamp: "2026-07-16T17:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:39Z" name: error-404-isvc-a90bb.18c2d6edc22a7f92 namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: de0b27ab-bded-4a97-a919-f66cae89f1a5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:45:16.848580Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:45:16Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d632fb8416fd namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: e9ba4d72-4597-4415-ad1b-cc95b47057a8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e62f1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d63308113f8a namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 4b777b25-3d14-41cd-bb90-41764957ff52 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:45:17Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d6333cf1fb69 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 421d0b95-29a0-4411-9150-9cc74b8b9cdc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d6333f06ca80 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 3c801895-7cd3-4edb-bba3-cd5d91825803 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" in 14.751s (14.751s including waiting). Image size: 1334629482 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d636ae4bb443 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: fd0ad9c6-36d8-4a7f-b599-6f8c2c8d5e80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d636b30c0ddf namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: 7e288ad4-c6aa-48c9-aceb-d6cd89aeb137 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d636b3924547 namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 1de7cf9b-2f23-4287-9cf9-ee9385311cc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d636b39f50f5 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 954154f1-e1db-42d3-b5a6-2c3839894239 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.493s (2.493s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:35Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d63748380049 namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 479345f1-8dd3-49cc-9c36-b422d134e41e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:35Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d6374ad27f58 namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 377fb643-7ffb-42a6-8f54-f015e193bf41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:45:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:35Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d6374b6a60cf namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 5c4309dd-5842-4e47-a39a-897d54994055 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:46:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:12Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d63773d651a2 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: 0a81f754-a3bc-4174-a99d-eec05c5fa91f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d647e513970f namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: 9671ae86-d2bc-4b2c-b80c-9eb85c9cec3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d647e513ff8e namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: ecf7f261-678f-46b0-8a94-0edf980b95f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c6ce7e22-0d4f-4cbc-82fb-d0a6096b0524 kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd.18c2d647fc2d6b49 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: afdaad38-9d0a-4ee3-8aff-c9d51eb662f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e62f1-predictor-57b4c448cd namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 734e7088-d1e2-4d95-81c9-f397dd96019c kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: 'Created pod: error-404-isvc-e62f1-predictor-57b4c448cd-lhtxd' metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: error-404-isvc-e62f1-predictor-57b4c448cd.18c2d632fb2375b9 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: cbb703eb-8da8-4e13-a26d-0f4bcf7685cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: ef2af06e-9223-45c6-8abf-022073f694d2 kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: Scaled up replica set error-404-isvc-e62f1-predictor-57b4c448cd from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: error-404-isvc-e62f1-predictor.18c2d632f87e81a4 namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 71502552-0eb9-46d9-895b-b29295ebce00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: baafd287-9420-4170-9078-15ed60d2179a kind: Event lastTimestamp: "2026-07-16T17:46:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:01Z" name: error-404-isvc-e62f1-predictor.18c2d63677281c94 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: f330caa0-cfe6-4333-ac41-4a693beed7f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: baafd287-9420-4170-9078-15ed60d2179a kind: Event lastTimestamp: "2026-07-16T17:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:01Z" name: error-404-isvc-e62f1-predictor.18c2d63677a5f7f3 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 2b9eabcd-1895-4baf-a187-547bc06aab15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: baafd287-9420-4170-9078-15ed60d2179a kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:31Z" name: error-404-isvc-e62f1-predictor.18c2d640f4100dc8 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: e5771399-1c90-4d78-aa16-244e9c5e8241 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: baafd287-9420-4170-9078-15ed60d2179a kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:31Z" name: error-404-isvc-e62f1-predictor.18c2d640f4a592e3 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: e8c66718-a1ed-4c83-ab35-99243ccd1ae8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e62f1 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 6735d7ce-8bfd-4b39-9bb9-ed92d49e5c81 kind: Event lastTimestamp: "2026-07-16T17:46:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:22Z" name: error-404-isvc-e62f1.18c2d632f87ec8d5 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: d59009a2-7c64-4871-9fa7-7d1f37e1cccf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e62f1 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 6735d7ce-8bfd-4b39-9bb9-ed92d49e5c81 kind: Event lastTimestamp: "2026-07-16T17:46:22Z" message: InferenceService [error-404-isvc-e62f1] is Ready metadata: creationTimestamp: "2026-07-16T17:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:22Z" name: error-404-isvc-e62f1.18c2d6423181e13b namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: c45d70ff-e830-479f-96bd-e7fd98cedd70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:12.337034Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-efe2d-predictor-568fd6d6fd-br76g to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d991f38406 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: cd5f1981-e119-4e54-99c0-b86c3a08712a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9ac1704af namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: ddb6187a-f9ef-4c2a-bdf5-ba4972d3b597 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9ae098326 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 07bff4d6-a8c8-4ee8-859c-a4c21848658d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9b0921ccd namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: 136b7299-532d-4bc3-9beb-d877951d8720 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9b102f496 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 29c37d62-bd52-4076-a78c-8801749d51b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9b10bd05e namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 281184cb-818e-4c48-a5bf-804e643465e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9b3c88929 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 59cdee9f-9275-4eb2-a2b1-a52ab769c8ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6d9b43c26b2 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: ca63cadd-57fc-4045-9299-c09731346f9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T17:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T18:06:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:30Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d6da223c50e3 namespace: kserve-ci-e2e-test resourceVersion: "25731" uid: a9644716-26fb-49cf-b6ed-9852cb60e503 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d75abfa852f7 namespace: kserve-ci-e2e-test resourceVersion: "25625" uid: 413fc2dc-4960-4c59-a52f-e0724ca66a76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:27Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d75abfa8cad4 namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 4b885bfb-7112-4145-bb9e-4acacc5fda24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 704b81b9-c1f6-45c8-9b9e-406696fcb9b5 kind: Event lastTimestamp: "2026-07-16T18:06:30Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:30Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g.18c2d75b97179c35 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: a36cd27b-2f64-46ee-870e-53d37a91113d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-efe2d-predictor-568fd6d6fd namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: f8d08129-283b-4a8b-adde-bda0e9a08fbf kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created pod: error-404-isvc-efe2d-predictor-568fd6d6fd-br76g' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor-568fd6d6fd.18c2d6d9914ad5f9 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: d0aeec3c-e46b-4186-85a8-392f8e2f14e5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 743992c7-a61b-4b3e-83de-a7ee9088c064 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Scaled up replica set error-404-isvc-efe2d-predictor-568fd6d6fd from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d-predictor.18c2d6d98f9c70dc namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 9e279b16-c630-4bfa-826a-af37922a143e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:57:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 576fb592-84c1-4e63-b9c4-d0d2c1e98085 kind: Event lastTimestamp: "2026-07-16T17:58:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:27Z" name: error-404-isvc-efe2d-predictor.18c2d6dd0e39d374 namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: 97dfe1b3-9122-427e-ab8b-80db0f2ba693 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:57:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 576fb592-84c1-4e63-b9c4-d0d2c1e98085 kind: Event lastTimestamp: "2026-07-16T17:58:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:27Z" name: error-404-isvc-efe2d-predictor.18c2d6dd0ebef286 namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: f1241b20-bd7d-4881-a72b-6eecf50479cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 576fb592-84c1-4e63-b9c4-d0d2c1e98085 kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:42Z" name: error-404-isvc-efe2d-predictor.18c2d6e08dc5ba71 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 519e142d-5606-4f77-90bd-0ea9933cabcd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 576fb592-84c1-4e63-b9c4-d0d2c1e98085 kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:42Z" name: error-404-isvc-efe2d-predictor.18c2d6e08e6b45d5 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 5b88c59d-e91b-4355-aedb-3ec27cbbef7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 1cc292f3-d655-40f0-9ab7-221bc91b3bf7 kind: Event lastTimestamp: "2026-07-16T17:58:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:00Z" name: error-404-isvc-efe2d.18c2d6d98f93b23e namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: c4a7b56e-bf60-40e3-a127-24dae52e885e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 1cc292f3-d655-40f0-9ab7-221bc91b3bf7 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-efe2d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-efe2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d.18c2d6d98ff326c4 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: c3f7d6a8-dcf0-4552-a319-40cc096967a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 1cc292f3-d655-40f0-9ab7-221bc91b3bf7 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-efe2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:12Z" name: error-404-isvc-efe2d.18c2d6d98ff3b06e namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: c0d95625-1602-469f-91cd-23405bf0e944 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 1cc292f3-d655-40f0-9ab7-221bc91b3bf7 kind: Event lastTimestamp: "2026-07-16T17:58:00Z" message: InferenceService [error-404-isvc-efe2d] is Ready metadata: creationTimestamp: "2026-07-16T17:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:00Z" name: error-404-isvc-efe2d.18c2d6e4de307660 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: a1fe8c62-6e67-461c-8042-38a01d7648e7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:46:46.876907Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d647f19f6ed4 namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 3c721b16-5f99-4451-95da-ca06340078f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f0d41-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d648019bf3df namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: bf568ce5-f018-4578-9b66-8027913f12a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d6482f8199f8 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 485290f3-e44a-4bff-b33e-e10ff5c9d05b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d64830d6ec82 namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 9bae07cb-a022-41f7-a557-d9a671696371 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d64833ac5391 namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: d6332fdd-eb88-4b97-ad81-696f9a8a63db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d6483410b3e3 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 1fb8d415-53af-4ce0-97a0-e6c84e6f9f4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d648341c696e namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: 25a4a391-4e8f-494d-bc52-dd034bcd8b43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:48Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d64836cf06c3 namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: b9ae253b-c64e-4a09-8f53-7fa82ac1ca25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:46:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:48Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d6483732982b namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 7d3e56d2-9560-4722-8242-b4bf43e35067 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:47:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:25Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d648839eb68d namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: d39a76a5-92da-4233-83f4-715d3b3b32e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:01Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d6c92aebc51a namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 81ae31d0-4727-4c92-8ccd-d583289fff82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: b8f32b5c-c76d-40a4-aaf4-a07591e24eed kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:01Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d.18c2d6c92aece7c2 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 80d3c63a-1228-4ee3-a55c-c05743b0487d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f0d41-predictor-9cfbdbc84 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ca13e020-8d1a-4ebd-97f2-bc7c55e0e995 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'Created pod: error-404-isvc-f0d41-predictor-9cfbdbc84-cjh4d' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-f0d41-predictor-9cfbdbc84.18c2d647f125381c namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 09dfd760-8831-4c7d-9dda-f6888d2e3d2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 2fa49606-5d39-4d71-a613-eca910c2d7aa kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Scaled up replica set error-404-isvc-f0d41-predictor-9cfbdbc84 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-f0d41-predictor.18c2d647ef85d96a namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 3870fd4e-af0a-4785-8245-f3a212ac0a9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 1feed06a-ae43-40a4-bd6c-7192e3ea9112 kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: error-404-isvc-f0d41-predictor.18c2d64b6e659436 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 91fbb1d7-f9df-4d0a-8578-e0bd9a8ee964 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 1feed06a-ae43-40a4-bd6c-7192e3ea9112 kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: error-404-isvc-f0d41-predictor.18c2d64b6eeb7a77 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: cb403257-ff24-4cdb-9816-c698843deb5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 1feed06a-ae43-40a4-bd6c-7192e3ea9112 kind: Event lastTimestamp: "2026-07-16T17:48:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:01Z" name: error-404-isvc-f0d41-predictor.18c2d6526c7e3f36 namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 8b586ff9-3f36-4f87-96a3-057f0a3c4c28 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 1feed06a-ae43-40a4-bd6c-7192e3ea9112 kind: Event lastTimestamp: "2026-07-16T17:48:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:01Z" name: error-404-isvc-f0d41-predictor.18c2d6526cfa810a namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 9fb040f4-b33d-45b6-a666-a577a6c6e5c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 293651ea-21f8-4a85-8c02-e0d79fe732c6 kind: Event lastTimestamp: "2026-07-16T17:47:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:35Z" name: error-404-isvc-f0d41.18c2d647ef81dff3 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 5c10ffd3-5332-451f-851a-5324810c7f93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 293651ea-21f8-4a85-8c02-e0d79fe732c6 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-f0d41": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f0d41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-f0d41.18c2d647efe89ee0 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: f66b6b7d-efdb-4fa4-8760-28dc23074f46 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 293651ea-21f8-4a85-8c02-e0d79fe732c6 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f0d41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:46Z" name: error-404-isvc-f0d41.18c2d647efe8de8c namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 1b2d4fca-9eaa-45e3-974f-ffdd9995d54a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 293651ea-21f8-4a85-8c02-e0d79fe732c6 kind: Event lastTimestamp: "2026-07-16T17:47:35Z" message: InferenceService [error-404-isvc-f0d41] is Ready metadata: creationTimestamp: "2026-07-16T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:35Z" name: error-404-isvc-f0d41.18c2d653412ca203 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: fe393556-32c1-45c9-b77c-7659e009e979 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:45:16.760214Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 to ip-10-0-141-151.ec2.internal metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:45:16Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d632f63fc9db namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 9efa51a9-baf6-4889-b8a1-e59c153aa79c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d63311337f77 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: e2c8c3ce-4a7a-4567-a059-b6c6072a30a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d633140e7a9d namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: 04d0c617-505c-4d14-94c6-09089a3cebe3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d6331825261f namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 92fc45d2-895d-4da0-84f7-2a27aefbe3c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d63318a4fd4c namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: 54700f4b-6ccc-49e9-b0e7-d122a92b937a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" metadata: creationTimestamp: "2026-07-16T17:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:21Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d6341d4bf2f0 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: dd70de5f-bcec-4c7b-bfe4-9fdcea338076 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" in 15.205s (15.205s including waiting). Image size: 1558048690 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:36Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d637a79bc039 namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 9d2b1a80-0438-45ba-9633-3f70e68886d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:36Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d637aa87d5f9 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 64bfce77-195f-4acf-a3d5-a8cbdef2c028 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:36Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d637aafe87b0 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: 86c865b2-d888-494c-9974-a150c01966a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:36Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d637ab0ac79e namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 0f87ef0e-5898-4720-afff-d5cea558e116 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 10.853s (10.853s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:47Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d63a31f53363 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: ea06f051-ff9f-45ed-916f-0a5f67ac0b95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:47Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d63a37776765 namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: a73c174f-aaf9-4c80-912d-5261d633f986 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:45:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:47Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d63a387ce391 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: a1f3f576-3be7-4260-8c7c-287b30699d15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:46:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:44Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d63a6ee15a06 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: 4ec0fbb8-1bbe-4012-9d8d-39aae37f5e32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:26Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d6513dc76549 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: a5612ccb-43c3-4c3c-9faf-6e980483dc8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:26Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d6513dc8c476 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: b4d2d113-76e0-4378-8651-ac8227134935 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: d10dbe18-5da2-481f-9579-f985ccb3e492 kind: Event lastTimestamp: "2026-07-16T17:47:29Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:29Z" name: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8.18c2d651f33e1b7c namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 8e319d3a-63be-470c-bcd4-3cde66a23088 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-d779bb8df namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: bf09fdbf-5c63-4c4c-b1db-daedfb56d971 kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-d779bb8df-bgvc8' metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: isvc-sklearn-graph-1-predictor-d779bb8df.18c2d632f5a6b037 namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: f2a54621-6035-4141-99f4-65246bc00c5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: e23c3fc5-c6ef-4669-8081-8ec44200f3b1 kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-d779bb8df from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: isvc-sklearn-graph-1-predictor.18c2d632f2c83366 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: fa6fc8b4-37a4-4669-8095-be609fb10bf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: c1139e23-1e6b-4a09-851a-e4f1304a39fd kind: Event lastTimestamp: "2026-07-16T17:46:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:16Z" name: isvc-sklearn-graph-1-predictor.18c2d636717196e4 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: f8e545cb-d997-4f68-a76f-8dac6001a787 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: c1139e23-1e6b-4a09-851a-e4f1304a39fd kind: Event lastTimestamp: "2026-07-16T17:46:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:16Z" name: isvc-sklearn-graph-1-predictor.18c2d63671f57e32 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 00231c5d-8d15-4695-bf63-63504b663821 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: c1139e23-1e6b-4a09-851a-e4f1304a39fd kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: isvc-sklearn-graph-1-predictor.18c2d63d6fe7f87d namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 01c8ea31-620c-41f0-a520-4f3e8b70c638 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: c1139e23-1e6b-4a09-851a-e4f1304a39fd kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: isvc-sklearn-graph-1-predictor.18c2d63d70a6e0c8 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: ef5263f2-80b2-4a2c-8ff5-2be866662800 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 10bcd44f-5759-4319-858e-2f3276feda44 kind: Event lastTimestamp: "2026-07-16T17:46:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:55Z" name: isvc-sklearn-graph-1.18c2d632f2f22573 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: d4352f61-bba2-4a8c-86f6-62797f56869b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 10bcd44f-5759-4319-858e-2f3276feda44 kind: Event lastTimestamp: "2026-07-16T17:46:54Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-16T17:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:54Z" name: isvc-sklearn-graph-1.18c2d649d3def29e namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 309e0113-2708-4fdd-a636-6284eaf4762f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:45:17.082351Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d63309733d7c namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: d74fed2e-946a-4a8a-a787-775d0354a5f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d63319293ae7 namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: e20a82bd-39c4-464a-b4fe-27c6389fa979 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:18Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:45:18Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d633532d55c3 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 4dae9465-eccd-4003-851a-17207fec9f0c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:18Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" metadata: creationTimestamp: "2026-07-16T17:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:18Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d633549e5d42 namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 50d185e6-fb8c-4e17-a003-9025a7732584 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:21Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" in 3.454s (3.454s including waiting). Image size: 301882321 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:21Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d634228bae48 namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 2a35f05f-9cd7-49ad-80f2-c8ce516db214 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:21Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d63425f17374 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 73303f29-b786-40ca-9595-af489637b430 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:21Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d63426626825 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 36820d37-304c-44aa-9cc7-3b7f45f2b511 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:26Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" metadata: creationTimestamp: "2026-07-16T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:26Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d6353d0e5150 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: a46de12a-03ee-4d8d-a329-84458a624179 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:39Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" in 12.793s (12.793s including waiting). Image size: 1558048690 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:39Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d63837993389 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 0bdfd2a4-4509-4fd7-9d49-5c9a9cc92848 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:39Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d6383b2ea2cb namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: eb69cd5c-3c95-4939-a470-94ebac329931 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:39Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d6383b9b1081 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: b23e4d88-085d-4739-b1f5-5b42d835ca9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:39Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:39Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d6383ba84489 namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: c82da530-8e84-45e2-9d7b-3c1238947041 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.191s (2.191s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:41Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d638be4639b6 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: baa2b2ee-717b-484d-be62-d938ced8c545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:41Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d638c16ddcc5 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 555511ba-3192-43f6-b2c0-c34042d00457 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:45:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:41Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d638c1de5543 namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 2e9a2de8-b396-4daa-989d-d6ad351d072b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:47:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:29Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d6393833ee82 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: a60b1c32-4cd7-4a88-a1ae-4c1f13c9d00e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d65149d29ec0 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 689c5bc6-f121-43ee-85f9-4f5db3eb8656 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d65149d33af4 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: ae3c06ab-db17-4325-9bc9-f25c16bec0f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: 7fcda0fa-aab6-41c4-86b1-d1c566a8477b kind: Event lastTimestamp: "2026-07-16T17:47:29Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:29Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh.18c2d651e67ee7fa namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 945e7285-b76f-431a-a2f5-8de88cc2182f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-c5d76d7f namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 90550699-672f-4d30-a6b7-6b6766fc2155 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-c5d76d7f-b9ckh' metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-2-predictor-c5d76d7f.18c2d6330903381f namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: d93c897f-c089-4732-8310-23988ec7ca30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: ef3116a1-e2ba-4d10-8c7c-16436f2a4358 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-c5d76d7f from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:17Z" name: isvc-sklearn-graph-2-predictor.18c2d633060c8286 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 1d328f2c-76ef-4c48-ae94-3acea07563a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: e9ca7849-23a2-4a53-b643-9d60f87d972d kind: Event lastTimestamp: "2026-07-16T17:45:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:47Z" name: isvc-sklearn-graph-2-predictor.18c2d63684c91e1f namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: b26fc57f-c64c-415e-a500-a5c756d44c6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: e9ca7849-23a2-4a53-b643-9d60f87d972d kind: Event lastTimestamp: "2026-07-16T17:45:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:47Z" name: isvc-sklearn-graph-2-predictor.18c2d63685552480 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 1f7f9438-e357-4906-9e36-e786e723a127 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:46:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: e9ca7849-23a2-4a53-b643-9d60f87d972d kind: Event lastTimestamp: "2026-07-16T17:47:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:17Z" name: isvc-sklearn-graph-2-predictor.18c2d63d8303eb2e namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: bc301da4-3ade-4231-a041-e142620416a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:46:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: e9ca7849-23a2-4a53-b643-9d60f87d972d kind: Event lastTimestamp: "2026-07-16T17:47:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:17Z" name: isvc-sklearn-graph-2-predictor.18c2d63d839dd4a6 namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: bac16c6b-8806-4154-bc18-dbe3e9f4e813 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: 93043668-0042-4169-b122-60eca9a878ef kind: Event lastTimestamp: "2026-07-16T17:46:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:59Z" name: isvc-sklearn-graph-2.18c2d6330614fece namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 859e6e92-aafe-43e2-a4a2-8cdde6f2c2f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15906" uid: 93043668-0042-4169-b122-60eca9a878ef kind: Event lastTimestamp: "2026-07-16T17:46:59Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-16T17:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:59Z" name: isvc-sklearn-graph-2.18c2d64aefd40a5d namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 7509cb5f-3df3-4aa4-96b4-595878250b1a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:45:16.965456Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-wvdwg to ip-10-0-141-151.ec2.internal metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:45:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d633027b9209 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: e8eb2da4-d210-4258-9427-0352ed9e41f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d6330ea30eef namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 180c573d-f219-475a-b42d-9a7bf017d497 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:18Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:45:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63340933ec5 namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: d26e8907-ff95-4656-9806-8399521c3c9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d633421b672f namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: c4714977-7522-48bf-817b-6d3e95d57a95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63344ff218d namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: abf7c27d-063d-4690-b5c0-46c54ec18706 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d6334588deed namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 9db1fd87-3433-4e97-9051-fb719cbc1418 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:22Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-16T17:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63459a92d04 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 8e1bf89d-8b3e-4957-a26b-4e426cc7097a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:47Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 25.171s (25.171s including waiting). Image size: 1306969339 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63a35f8fc8f namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 0a951ec9-f722-4e7d-bebe-7c1541e9c275 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63a3c932955 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 276d8d41-731e-4ab3-bf1b-ef51effd641c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63a3dd91bce namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 50cf1a26-b0f7-4942-ae29-77b758f43771 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63a3dee8517 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 5e37ac28-7077-4921-bb97-a737a0f0ad64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63a43c15d12 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 956d2d38-1ff8-499e-a56f-b12d63bb024b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:45:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63a4510a97b namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: d571540a-9ca4-46f6-a8db-6c197d2d6188 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T17:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d63aaad5bf8e namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 8193196b-83b5-4e32-8acd-6432bc20f0b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d65140e92576 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 506e4456-1957-404b-925b-65cbb172a3ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: fbf50dfb-21f7-4f35-8496-a8cfaa013804 kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-wvdwg.18c2d65140e9be61 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 9e57bf24-3bd9-4d96-8659-0308ea0b7a5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-151.ec2.internal source: component: kubelet host: ip-10-0-141-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14691" uid: 5105fbeb-01f6-41d9-8834-1231cca2c9ba kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-wvdwg' metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c2d63301e73135 namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 7d3b5cf2-287d-4155-a121-7e25bfcc7270 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: c90a69a5-0c81-4a5d-a077-8fe66e783ce9 kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: isvc-xgboost-graph-predictor.18c2d632ff1fc94b namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: 3359178d-4819-4e59-9771-93de92708a8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d58246d9-b97a-4b3a-b453-8ea942feaede kind: Event lastTimestamp: "2026-07-16T17:46:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:16Z" name: isvc-xgboost-graph-predictor.18c2d6367df66df0 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 8571ca99-efb1-4bf3-84e3-6af433d642e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d58246d9-b97a-4b3a-b453-8ea942feaede kind: Event lastTimestamp: "2026-07-16T17:46:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:16Z" name: isvc-xgboost-graph-predictor.18c2d6367e6c8c03 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 5c2233c5-2437-4fbe-8c3b-a3b012ce3089 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: d58246d9-b97a-4b3a-b453-8ea942feaede kind: Event lastTimestamp: "2026-07-16T17:47:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:02Z" name: isvc-xgboost-graph-predictor.18c2d64479a4f7c5 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: a5e30e19-fc89-4ea3-8a0a-b62ebf5d1f04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: d58246d9-b97a-4b3a-b453-8ea942feaede kind: Event lastTimestamp: "2026-07-16T17:47:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:02Z" name: isvc-xgboost-graph-predictor.18c2d6447a1a6c8a namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 6ed03583-ed38-45ef-9ba5-067e8f296a56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 7cb909bc-a505-4f65-ab1c-2c0b8fcb76ea kind: Event lastTimestamp: "2026-07-16T17:46:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:45Z" name: isvc-xgboost-graph.18c2d632ff4efb86 namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 3db1c7a9-88ed-442a-a0d5-759ae5c8ee8a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 7cb909bc-a505-4f65-ab1c-2c0b8fcb76ea kind: Event lastTimestamp: "2026-07-16T17:46:45Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-16T17:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:45Z" name: isvc-xgboost-graph.18c2d647baba6000 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: d0d5636b-30cf-44f6-8c17-8e17fe79dce9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:47:16.536470Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-c9f7f4f47-wwvjz to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:47:16Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d64ed9786134 namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 144e750e-e5cc-46d8-aa84-f21ab83fb5b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:47:16Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d64ef2c56950 namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 39a0ff3f-5c04-47f8-ba93-dcc92a5ca5cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:16Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d64ef4163165 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: c5cba919-71b3-4cba-85ef-2b31096a5f8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: "2026-07-16T17:47:17Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-16T17:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:17Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d64ef6f9febf namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 998a4c08-9f88-45f7-a1f0-1589e23ff24c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: "2026-07-16T17:47:17Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-16T17:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:17Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d64ef75c9b6f namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 6ee2bc9d-2732-45ff-85f8-86dba70358ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:26Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d651338f864e namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: f31f2794-6e47-495a-86d5-69e9d403003e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c9f7f4f47-wwvjz namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 2b116ced-7b89-49c4-9f18-5d001761774a kind: Event lastTimestamp: "2026-07-16T17:47:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:53Z" name: model-chainer-c9f7f4f47-wwvjz.18c2d651894ba74e namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 425a32e7-39a0-4fe1-bcd4-282fec558225 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-c9f7f4f47 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: dd7c9fe4-6257-4d54-92ce-752c1e88dd2d kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'Created pod: model-chainer-c9f7f4f47-wwvjz' metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: model-chainer-c9f7f4f47.18c2d64ed88bb8d8 namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: c085a399-efcc-4d6c-b487-15cf2f53a9b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 092572b9-76cc-49dc-a221-e7cd37b6a779 kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: Scaled up replica set model-chainer-c9f7f4f47 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: model-chainer.18c2d64ed7081899 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 2d852a81-57f3-4116-bb0f-3c6f15520c9f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 7c451118-77d2-450f-b018-6613a2760b74 kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:16Z" name: model-chainer.18c2d64ede347143 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: d6105c10-4112-47bc-a353-752e715ff265 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:48:26.852342Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-38e88-5fb9579648-khh64 to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:48:26Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d65f389f7acf namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 8dc62a7c-bfdb-4c13-a24a-506287a81270 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: "2026-07-16T17:48:27Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:48:27Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d65f525dbc49 namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: 71d55f37-0ad6-4044-8c8a-ae267ce1bca3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-38e88} kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: "2026-07-16T17:48:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:27Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d65f53b9cb55 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 4ab8ac60-acf2-4365-9a94-19b32de256ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-38e88} kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: "2026-07-16T17:48:27Z" message: 'Created container: sequence-graph-38e88' metadata: creationTimestamp: "2026-07-16T17:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:27Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d65f56c1d444 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 4e58cbce-f3fa-46ab-939a-df1e6df643ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-38e88} kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: "2026-07-16T17:48:27Z" message: Started container sequence-graph-38e88 metadata: creationTimestamp: "2026-07-16T17:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:27Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d65f5735271a namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 14c842a1-418d-4f53-9c7e-68561451a883 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-38e88} kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Stopping container sequence-graph-38e88 metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:41Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d6d2620d5e96 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: 8aac99de-5673-4a35-8f34-a870b51bd6b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-38e88} kind: Pod name: sequence-graph-38e88-5fb9579648-khh64 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 403e454a-70f5-4888-8482-66cc1742652e kind: Event lastTimestamp: "2026-07-16T17:57:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:08Z" name: sequence-graph-38e88-5fb9579648-khh64.18c2d6d2d59a9da3 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 58608c70-3517-48ad-8b5f-706339a078e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-38e88-5fb9579648 namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 9e763fba-63fa-4f80-a474-697bb52a3041 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: 'Created pod: sequence-graph-38e88-5fb9579648-khh64' metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:26Z" name: sequence-graph-38e88-5fb9579648.18c2d65f383c38d3 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 14a2ff90-846b-40dc-838f-5e9e632bc089 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: b1f97936-79ab-458c-8151-be72ba17b865 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: Scaled up replica set sequence-graph-38e88-5fb9579648 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:26Z" name: sequence-graph-38e88.18c2d65f375b9897 namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 4f647b7f-d6a7-4869-b8d7-5c731f2952c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: c298fc75-8156-45d3-bf1d-8ee928e8d77e kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: InferenceGraph [sequence-graph-38e88] is Ready metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:26Z" name: sequence-graph-38e88.18c2d65f3c7b3f82 namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 4ccb1672-e4ae-4625-b241-b0cb12aa2c66 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 351961f6-8ea9-4e82-a0fe-fd825abbcb55 kind: Event lastTimestamp: "2026-07-16T17:48:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:56Z" name: sequence-graph-38e88.18c2d662b5e0a831 namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: 282d7deb-4249-4c38-ab13-204a4abed3a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 351961f6-8ea9-4e82-a0fe-fd825abbcb55 kind: Event lastTimestamp: "2026-07-16T17:48:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:56Z" name: sequence-graph-38e88.18c2d662b64f804d namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: deb4fc59-8c67-4d69-87f6-1ad162ae8a7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:41.681263Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6687f-85bb45f787-r8tvt to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:41Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e06700fd31 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: 4aa655de-ff92-4fc0-83bb-c4d58b656e81 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6687f-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:41Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e07338d2b0 namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 20e45fbd-80af-481d-bfef-e00a962fd779 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:42Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e0a447d461 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: ad39b10c-2b88-4afa-b311-92c7319f45f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6687f} kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:42Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e0a59bba30 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 04bca4de-cd51-4b0b-825a-a2999d110f11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6687f} kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: 'Created container: sequence-graph-6687f' metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:42Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e0a93454b9 namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 32042f3b-35f6-41c7-aa99-64dd344cdcde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6687f} kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: Started container sequence-graph-6687f metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:42Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e0a9a96f40 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: c2157b19-c9a7-445a-8264-b510f4d2df57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6687f} kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: Stopping container sequence-graph-6687f metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:51Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e2c0c3afa9 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: ef8c5c63-169c-4c65-8d54-cd46ee39ed24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:57:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6687f} kind: Pod name: sequence-graph-6687f-85bb45f787-r8tvt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9f1b324f-0947-4543-8f2c-d0bb33ae79c1 kind: Event lastTimestamp: "2026-07-16T17:58:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:18Z" name: sequence-graph-6687f-85bb45f787-r8tvt.18c2d6e344174136 namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 70a958a2-bb95-4841-9fae-4f586b859266 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6687f-85bb45f787 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: e3d257a7-89b9-4e9c-b9f3-7733d811be8d kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: 'Created pod: sequence-graph-6687f-85bb45f787-r8tvt' metadata: creationTimestamp: "2026-07-16T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:41Z" name: sequence-graph-6687f-85bb45f787.18c2d6e066927b26 namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: e9a94f59-d8e0-41f4-b3e5-48f7743a3eb3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6687f namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 4b75c334-71bc-4945-bc92-63ae570ac510 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: Scaled up replica set sequence-graph-6687f-85bb45f787 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:41Z" name: sequence-graph-6687f.18c2d6e065cd6902 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: 16b3f02d-4f54-4021-949d-305d181c34f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6687f namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 84ccc3f6-ca71-4c34-9792-f5ccd62eb004 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: InferenceGraph [sequence-graph-6687f] is Ready metadata: creationTimestamp: "2026-07-16T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:41Z" name: sequence-graph-6687f.18c2d6e06b9b8b06 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 98dc437f-5ab2-4245-a193-d1678ce7a128 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:58:52.005936Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a90bb-5b767cc4d9-qrwl6 to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d6f0c6ae49f9 namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: fd568562-86cc-461d-8dda-e8306b7b4478 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22143" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d6f0e056f774 namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: 4c2fe1bc-b7e1-473b-b1ea-1fabde5a3a39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a90bb} kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d6f0e195c5dc namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: 463a584f-ef66-4b67-9d49-bb813a884e8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a90bb} kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: 'Created container: sequence-graph-a90bb' metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d6f0e4a7908c namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 71f68666-46e6-41b9-a8aa-7cdd42262d35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a90bb} kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Started container sequence-graph-a90bb metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d6f0e514b645 namespace: kserve-ci-e2e-test resourceVersion: "22162" uid: 90d4f990-37ff-496d-9c25-57ff3c31186a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a90bb} kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Stopping container sequence-graph-a90bb metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d763ecbfd10c namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 8ac36142-3464-4dc2-a9a8-87583e9aeac3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a90bb} kind: Pod name: sequence-graph-a90bb-5b767cc4d9-qrwl6 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 6a1a1886-b7e2-4b8e-9812-feb2615632a8 kind: Event lastTimestamp: "2026-07-16T18:07:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T18:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:34Z" name: sequence-graph-a90bb-5b767cc4d9-qrwl6.18c2d76490843f28 namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: f51eff4e-001c-47fa-824c-2977a641cc16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a90bb-5b767cc4d9 namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: 2bc02376-17d7-4d52-9e00-6a03e633dd79 kind: Event lastTimestamp: "2026-07-16T17:58:51Z" message: 'Created pod: sequence-graph-a90bb-5b767cc4d9-qrwl6' metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb-5b767cc4d9.18c2d6f0c64ea480 namespace: kserve-ci-e2e-test resourceVersion: "22133" uid: 13304b0e-9d90-427a-8a5e-9359dda19f85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: 45651a64-6d20-48be-a9ae-5bec5b5bb10a kind: Event lastTimestamp: "2026-07-16T17:58:51Z" message: Scaled up replica set sequence-graph-a90bb-5b767cc4d9 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:51Z" name: sequence-graph-a90bb.18c2d6f0c53ff045 namespace: kserve-ci-e2e-test resourceVersion: "22129" uid: b6a486f4-fd17-4cd2-b816-05d4d20ada86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: c7b888af-753d-4222-a0e3-4cba41b41b2b kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: InferenceGraph [sequence-graph-a90bb] is Ready metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:52Z" name: sequence-graph-a90bb.18c2d6f0caff7b36 namespace: kserve-ci-e2e-test resourceVersion: "22148" uid: 4ba27993-c34e-4af1-9912-96b17153ad8f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: e009b975-4b5e-46e3-829c-291b02fcc6f0 kind: Event lastTimestamp: "2026-07-16T17:59:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:06Z" name: sequence-graph-a90bb.18c2d6f444125b47 namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: 89749749-4906-432e-9f32-387607a6a530 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:59:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: e009b975-4b5e-46e3-829c-291b02fcc6f0 kind: Event lastTimestamp: "2026-07-16T17:59:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:07Z" name: sequence-graph-a90bb.18c2d6f444c08325 namespace: kserve-ci-e2e-test resourceVersion: "22272" uid: fad047ca-f60a-4aaa-ab10-6e4e4bc27f3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:59:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: e009b975-4b5e-46e3-829c-291b02fcc6f0 kind: Event lastTimestamp: "2026-07-16T17:59:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:22Z" name: sequence-graph-a90bb.18c2d6f7c46e9f61 namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: d4d2f111-3a74-474e-97cf-bf086e6f31d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:59:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: e009b975-4b5e-46e3-829c-291b02fcc6f0 kind: Event lastTimestamp: "2026-07-16T17:59:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:22Z" name: sequence-graph-a90bb.18c2d6f7c4fda44d namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: bba58bb6-b0c9-4800-8ec8-dc16fe0d2aff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:08:37.596804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-21cc5-6dc7d7677b-phnvq to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T18:08:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:08:37Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d7791e9dd990 namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: bd08ea67-166a-4748-a596-0c06411d6a12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:38Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27184" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: "2026-07-16T18:08:38Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:08:38Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d77938832356 namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: 64886b35-70d4-44d2-af51-78160141f03e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-21cc5} kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: "2026-07-16T18:08:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T18:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:38Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d77939b9ac5a namespace: kserve-ci-e2e-test resourceVersion: "27200" uid: 67a64644-5b71-4da1-9415-8d2de2e2e6fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-21cc5} kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: "2026-07-16T18:08:38Z" message: 'Created container: splitter-graph-21cc5' metadata: creationTimestamp: "2026-07-16T18:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:38Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d7793ca391bc namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: 95f16bd1-705a-4d72-87ed-b7bfb446a4a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-21cc5} kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: "2026-07-16T18:08:38Z" message: Started container splitter-graph-21cc5 metadata: creationTimestamp: "2026-07-16T18:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:38Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d7793d6b36c5 namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: 494e9971-2e75-40c8-b216-efee1f96691a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-21cc5} kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: "2026-07-16T18:16:52Z" message: Stopping container splitter-graph-21cc5 metadata: creationTimestamp: "2026-07-16T18:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:52Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d7ec4bb40224 namespace: kserve-ci-e2e-test resourceVersion: "30645" uid: e9db77e9-fb77-4766-8d29-fcf44678cfe4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-21cc5} kind: Pod name: splitter-graph-21cc5-6dc7d7677b-phnvq namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 3be09695-24b0-46d7-92f5-bc651a5e0d9d kind: Event lastTimestamp: "2026-07-16T18:17:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T18:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:17:20Z" name: splitter-graph-21cc5-6dc7d7677b-phnvq.18c2d7eceff233a0 namespace: kserve-ci-e2e-test resourceVersion: "30887" uid: a5687e61-b79f-4d7f-b2fc-2e4dc05bcf3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-21cc5-6dc7d7677b namespace: kserve-ci-e2e-test resourceVersion: "27172" uid: d51c7663-ead8-4a51-80b4-9872b7e7c3fa kind: Event lastTimestamp: "2026-07-16T18:08:37Z" message: 'Created pod: splitter-graph-21cc5-6dc7d7677b-phnvq' metadata: creationTimestamp: "2026-07-16T18:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:37Z" name: splitter-graph-21cc5-6dc7d7677b.18c2d7791e328048 namespace: kserve-ci-e2e-test resourceVersion: "27176" uid: 006f2cac-5202-4a7d-a88a-c9c965a6f5f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 90edf590-c0bf-4607-a781-2360dc7c3704 kind: Event lastTimestamp: "2026-07-16T18:08:37Z" message: Scaled up replica set splitter-graph-21cc5-6dc7d7677b from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:37Z" name: splitter-graph-21cc5.18c2d7791d420259 namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: 124c4f07-dfd4-4736-aa1f-717e180c5b61 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "27187" uid: 914a9949-90de-4901-a748-a1f29550e123 kind: Event lastTimestamp: "2026-07-16T18:08:37Z" message: InferenceGraph [splitter-graph-21cc5] is Ready metadata: creationTimestamp: "2026-07-16T18:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:08:37Z" name: splitter-graph-21cc5.18c2d779232ac4b1 namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: 627a7742-448b-450f-99d0-938ce43ba0a6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T18:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: 6a449613-be38-4a98-9d2d-90cda2c4b951 kind: Event lastTimestamp: "2026-07-16T18:09:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:09:22Z" name: splitter-graph-21cc5.18c2d77c9ba06efd namespace: kserve-ci-e2e-test resourceVersion: "27528" uid: e3cddc8c-515a-4ffe-9c3c-963f3e34ab34 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T18:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: 6a449613-be38-4a98-9d2d-90cda2c4b951 kind: Event lastTimestamp: "2026-07-16T18:09:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:09:22Z" name: splitter-graph-21cc5.18c2d77c9bea1389 namespace: kserve-ci-e2e-test resourceVersion: "27529" uid: 80e02fe2-511f-4405-9654-35e949aa01a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:07:27.238553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26354" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a019b-69676cb5c4-5td7b to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d768bcf020b4 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: a20d5042-410a-4a3b-a06a-b17365393073 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d768d68da131 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: a4c3b4c3-7035-4ffc-b39b-679e25d5dae2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a019b} kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d768d7ed4ecd namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: f1d78699-d09b-4550-a863-14308bb5a3b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a019b} kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: 'Created container: splitter-graph-a019b' metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d768daeddf36 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: ad4e3dd9-8b54-4bd9-8d6c-0eb5e5a4b060 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a019b} kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: Started container splitter-graph-a019b metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d768db5fb456 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: c323de80-f132-46d0-b5ec-c85cf1aab90e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a019b} kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Stopping container splitter-graph-a019b metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:37Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d76b1643addb namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: c0541146-0deb-4c7c-af6a-2f8b64d0c93b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a019b} kind: Pod name: splitter-graph-a019b-69676cb5c4-5td7b namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 072639f0-8e17-4af7-925d-c3d62da94102 kind: Event lastTimestamp: "2026-07-16T18:08:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T18:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:04Z" name: splitter-graph-a019b-69676cb5c4-5td7b.18c2d76ba3677f2d namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: debddf57-f5ba-44b0-86f5-7b4bd74a53ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a019b-69676cb5c4 namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: 59507f3e-7db2-44e5-b217-781eb000f25f kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: 'Created pod: splitter-graph-a019b-69676cb5c4-5td7b' metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b-69676cb5c4.18c2d768bc703574 namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: 8a3e67b4-47e3-4113-9be4-bf5cbfc77844 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a019b namespace: kserve-ci-e2e-test resourceVersion: "26349" uid: f59991a9-55c4-4acb-ac0d-8849b5cdde29 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: Scaled up replica set splitter-graph-a019b-69676cb5c4 from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b.18c2d768bba495a5 namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: 0c20b208-05e7-41c4-a493-042ee866e967 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a019b namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 8240306b-5fce-4379-b610-01d845c5f25b kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: InferenceGraph [splitter-graph-a019b] is Ready metadata: creationTimestamp: "2026-07-16T18:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:27Z" name: splitter-graph-a019b.18c2d768c1a27ba0 namespace: kserve-ci-e2e-test resourceVersion: "26367" uid: 3541de27-df56-4798-991b-8fea3a99b32f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:56:01.864259Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1da67-predictor-76bff85dd9-db9wj to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:56:01Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c929725421 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 05bd4f84-d255-46eb-af3c-133807eb8e62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1da67-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:02Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c933d14558 namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: a86cb21e-d505-493d-a58d-1e3ebca42519 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:02Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:56:02Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c9670381bd namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: c460ee6c-8f62-4b46-8de5-9754a68a6b5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:02Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c9687132a6 namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: 56ed7810-2b2a-4980-848a-921b4c9739c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:02Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c96c876b07 namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: b5686c57-aa83-4a80-bf64-8f02a8443ba0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c96d2d5db1 namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: f3a2fa13-d2f3-4ad6-a30e-a6f0e51330e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c96d38493b namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: ccdd29f7-38af-4e47-b551-bda621f1a5cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c9700af85f namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: 5d13e06b-c426-4809-bfe0-4d5b03c77b60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:03Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c97080c982 namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 06f20e54-11d6-467e-9562-1e2130470e9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:56:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:39Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6c9881829f4 namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 29c57fef-ddfc-4e71-87b6-798fd682b821 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6d98750b7bb namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 3f4ece90-46b8-4d04-b53e-def7402243ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6d987512143 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: 458b013a-79a3-44f4-a68e-6d173e19043a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: d9452ec7-5072-4e69-ab92-fd9c40131e0a kind: Event lastTimestamp: "2026-07-16T17:57:14Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:14Z" name: success-200-isvc-1da67-predictor-76bff85dd9-db9wj.18c2d6da103dce63 namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: 084590ef-9f7d-443e-aa1c-b0391217f4ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1da67-predictor-76bff85dd9 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 33801a9b-c3aa-4ecc-b4b1-65df97635f91 kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: 'Created pod: success-200-isvc-1da67-predictor-76bff85dd9-db9wj' metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:01Z" name: success-200-isvc-1da67-predictor-76bff85dd9.18c2d6c92961fba0 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: e02f007e-b5b1-42ca-b51c-ee4a4bd4569f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 2fcddb12-9648-487d-90ac-937ebae01226 kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Scaled up replica set success-200-isvc-1da67-predictor-76bff85dd9 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:01Z" name: success-200-isvc-1da67-predictor.18c2d6c925fd8b4a namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: ac4185d8-208f-4fea-bb71-1350627330f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 6eaad486-47e7-4be1-b522-b3392a25802a kind: Event lastTimestamp: "2026-07-16T17:56:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:31Z" name: success-200-isvc-1da67-predictor.18c2d6cca484864d namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 810aca05-5e93-4360-b66b-4155b1db5b8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 6eaad486-47e7-4be1-b522-b3392a25802a kind: Event lastTimestamp: "2026-07-16T17:56:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:31Z" name: success-200-isvc-1da67-predictor.18c2d6cca4f46d20 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 283455e1-2dd4-47b3-80ec-d494358ceecb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 6eaad486-47e7-4be1-b522-b3392a25802a kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:01Z" name: success-200-isvc-1da67-predictor.18c2d6d3a2cfe7ad namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: f6a2786f-36bb-43bf-bc17-9bdd42477126 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1da67-predictor namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 6eaad486-47e7-4be1-b522-b3392a25802a kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:01Z" name: success-200-isvc-1da67-predictor.18c2d6d3a3316efd namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: cbf7e746-60d1-4719-b821-6128d2d0a008 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1da67 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 5fd81627-3c17-4d30-bdee-85a123e467f2 kind: Event lastTimestamp: "2026-07-16T17:56:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:49Z" name: success-200-isvc-1da67.18c2d6c925e0ca7c namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 245c1dd2-39cb-4d28-91f9-515bc05eaf50 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1da67 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 5fd81627-3c17-4d30-bdee-85a123e467f2 kind: Event lastTimestamp: "2026-07-16T17:56:49Z" message: InferenceService [success-200-isvc-1da67] is Ready metadata: creationTimestamp: "2026-07-16T17:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:49Z" name: success-200-isvc-1da67.18c2d6d4467c0a7f namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 8196fff3-796a-45a0-bcc8-afe3fe60ea46 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:07:37.524114Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-21cc5-predictor-76676b5555-2rb8n to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b22015bc7 namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: b364e94a-8145-479d-bd54-da93ac54696f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-21cc5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b2de8be33 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 0fc53dde-dd13-404c-b507-a75314cde1ed reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b5f937058 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: d9743ec9-5653-4b63-97dc-48ed67a2cbfe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b60e2d93b namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: f268e81b-7054-4ca4-9bc3-f562e5976550 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b644844a9 namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: b4d78188-a652-41ed-ba22-07534ec6d590 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b64be5cf4 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 32d7fff7-93a9-493e-91bc-48d8f4b3ac74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b64c7d7f7 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 5612a0e1-809b-4544-b262-19d25eb192be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b682eafa6 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 8ac85020-7dc0-403a-a559-e25f9320a30e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:07:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:38Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b68a63e3a namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: 75ca52be-6cb6-48d7-91ae-82c77b4a2b9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T18:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:08:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:16Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d76b874a63ee namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: c56b254a-dbfa-4cb2-b5a6-ecf833419e32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:16:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:52Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d7ec5297a482 namespace: kserve-ci-e2e-test resourceVersion: "30662" uid: 8f61433d-2c68-4be8-9120-91d6de8f19ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: c2f85123-b57b-4560-b655-029db7a3ce32 kind: Event lastTimestamp: "2026-07-16T18:16:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:16:52Z" name: success-200-isvc-21cc5-predictor-76676b5555-2rb8n.18c2d7ec529844ba namespace: kserve-ci-e2e-test resourceVersion: "30663" uid: b01f9437-2780-4d6d-9a5b-856ae95be6ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-21cc5-predictor-76676b5555 namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: cbc9422a-47b2-4b07-9c21-0abc6ceb4c97 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'Created pod: success-200-isvc-21cc5-predictor-76676b5555-2rb8n' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-21cc5-predictor-76676b5555.18c2d76b219733d7 namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: 5e792574-ccc1-4dab-b3c9-c5b5c8629abf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: c56434f1-d2fd-4233-ada9-836e463f2c00 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Scaled up replica set success-200-isvc-21cc5-predictor-76676b5555 from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-21cc5-predictor.18c2d76b1f9e6feb namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 0de76994-aa4a-4591-b264-0e758c4f2a15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T18:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26525" uid: 4eb57b6f-383d-47cc-aa7e-562dc8ca186b kind: Event lastTimestamp: "2026-07-16T18:08:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:22Z" name: success-200-isvc-21cc5-predictor.18c2d76e9e4499c8 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 717caa41-246e-4f77-a577-ece3baa217dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T18:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26525" uid: 4eb57b6f-383d-47cc-aa7e-562dc8ca186b kind: Event lastTimestamp: "2026-07-16T18:08:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:22Z" name: success-200-isvc-21cc5-predictor.18c2d76e9eaa99d0 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: ae954295-2730-4079-9213-c3bd2dc3e844 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:08:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: 4eb57b6f-383d-47cc-aa7e-562dc8ca186b kind: Event lastTimestamp: "2026-07-16T18:08:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:52Z" name: success-200-isvc-21cc5-predictor.18c2d7791bdb9e98 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: 1a2997bc-19bd-4690-a753-ce024d779261 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:08:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21cc5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: 4eb57b6f-383d-47cc-aa7e-562dc8ca186b kind: Event lastTimestamp: "2026-07-16T18:08:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:52Z" name: success-200-isvc-21cc5-predictor.18c2d7791c548e8e namespace: kserve-ci-e2e-test resourceVersion: "27309" uid: feaba1f7-0d7c-48a0-bc5f-9e3a9b3aa007 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: c2709936-5533-4716-bd8c-5f7388341ad6 kind: Event lastTimestamp: "2026-07-16T18:08:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:08:26Z" name: success-200-isvc-21cc5.18c2d76b1f9f3f1b namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: c046d396-364f-412c-82b4-fc5a4a86680c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: c2709936-5533-4716-bd8c-5f7388341ad6 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-21cc5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-21cc5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-21cc5.18c2d76b20228a77 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 4ed74b4e-6272-4b34-9c9d-0749cc566f7d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: c2709936-5533-4716-bd8c-5f7388341ad6 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-21cc5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-21cc5.18c2d76b2022d607 namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: 1d6732fe-d934-402b-8e25-21c1f82c8e15 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21cc5 namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: c2709936-5533-4716-bd8c-5f7388341ad6 kind: Event lastTimestamp: "2026-07-16T18:08:26Z" message: InferenceService [success-200-isvc-21cc5] is Ready metadata: creationTimestamp: "2026-07-16T18:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:08:26Z" name: success-200-isvc-21cc5.18c2d7767eedd7ae namespace: kserve-ci-e2e-test resourceVersion: "27077" uid: 74524185-2241-4c1b-a379-f137511126c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:47:26.815126Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-38e88-predictor-5d744d6567-8cj6h to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:47:26Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d6513e204c30 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 2400dea9-0ed4-4538-9e65-27120e6665c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-38e88-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:27Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d65145d83486 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 46c31bec-47f0-428e-8f2b-b66b679bcfb5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b0c62a6d namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: ae19564b-e822-4227-8ecc-1ff90cdcaa09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b2138ade namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 309c335d-0060-410b-9aaa-31a8b98d285e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b54cc053 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 4997ed3b-03f7-4802-9bec-e471e3651128 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b5d9057b namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 75bf7852-4e38-4a81-8568-4c457387e7c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b5e2d5b3 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: e3afab31-e83e-4d65-b7fb-c79cbef7c3e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b82a7a8c namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 58e1ab00-669b-49e8-9b85-b7e318885677 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:47:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:28Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d651b88baadd namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 16cd3f65-41a7-4508-ab18-4580e30e1a90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:48:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:06Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d65218f9b6bb namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 7ce851fc-cf5f-459e-af56-1734103fda0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d6d26bb8ef59 namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: eb9b39d9-8e70-4edc-9d47-8e5932a2767f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 536a3642-b49f-40f3-97f8-9c325a55a02d kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-38e88-predictor-5d744d6567-8cj6h.18c2d6d26bb95894 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 8dbde8be-e042-4580-8e99-1bbfbed97b20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-38e88-predictor-5d744d6567 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 6224341c-1e7b-48fc-a5ca-0137a2e7b33f kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: 'Created pod: success-200-isvc-38e88-predictor-5d744d6567-8cj6h' metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:26Z" name: success-200-isvc-38e88-predictor-5d744d6567.18c2d6513db7f525 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9e65d01-1fc6-4e24-a647-a265e43418c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: e1cf3336-bcff-4270-a20d-764c1670a5ec kind: Event lastTimestamp: "2026-07-16T17:47:26Z" message: Scaled up replica set success-200-isvc-38e88-predictor-5d744d6567 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:26Z" name: success-200-isvc-38e88-predictor.18c2d6513baef972 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 28e42449-cb41-45b7-b2b5-26998341c4a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 7ff7bc84-ff8d-4096-95ca-6a4831a84cae kind: Event lastTimestamp: "2026-07-16T17:47:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:56Z" name: success-200-isvc-38e88-predictor.18c2d654ba30aae5 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 71691f4f-04f3-4021-8cb1-24fc34772990 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 7ff7bc84-ff8d-4096-95ca-6a4831a84cae kind: Event lastTimestamp: "2026-07-16T17:47:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:56Z" name: success-200-isvc-38e88-predictor.18c2d654baae6181 namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: 7316dca3-2370-4655-90d2-987d8936db67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 7ff7bc84-ff8d-4096-95ca-6a4831a84cae kind: Event lastTimestamp: "2026-07-16T17:48:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:41Z" name: success-200-isvc-38e88-predictor.18c2d65bb8e1a286 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: ecbc6ffe-5416-40c0-b57e-6782e422687c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38e88-predictor namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 7ff7bc84-ff8d-4096-95ca-6a4831a84cae kind: Event lastTimestamp: "2026-07-16T17:48:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:41Z" name: success-200-isvc-38e88-predictor.18c2d65bb956e1e5 namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: d8e2b0c4-9560-4679-b4c0-04dff318de30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:47:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 11f27369-fee0-4fd8-bf3b-cf9d0ef37560 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:16Z" name: success-200-isvc-38e88.18c2d6513bad890c namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 9f9d0611-6d80-44b9-ad79-f58904a9f811 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-38e88 namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: 11f27369-fee0-4fd8-bf3b-cf9d0ef37560 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: InferenceService [success-200-isvc-38e88] is Ready metadata: creationTimestamp: "2026-07-16T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:16Z" name: success-200-isvc-38e88.18c2d65cd44612d0 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 29168830-9700-4b63-9463-e7ce1968f882 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:56:41.669925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6687f-predictor-769fddcb6f-tfzgl to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d26e0c9c81 namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b4c0529a-e2c5-4387-bc48-d26b7e6876d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d287f589c4 namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 0157b315-4390-4127-83f9-9c434bd61fb9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d2897040ff namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: c675dc70-0d8b-45d1-a1bd-1768106c4747 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d28d3ace47 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 50a4da14-7652-4645-8898-b9c7998c2e4b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d28dbe8aef namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 50359aee-70ae-42f5-8ebb-a6681c7e929f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d28dcb0fa1 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 5243397c-4e55-4a5d-90e6-198cf8ad9335 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d2910ed665 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: bdf699dc-2051-47bd-a9d1-b9f5575b3e34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:56:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:42Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d291ae5b48 namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: c0e95d9e-f32b-4d69-8abb-12c4b18be860 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:57:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:18Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6d2a662d37f namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 783ae6aa-5e8d-4d59-a70d-1fd7e960c65a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6e2c82405d8 namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 5aa4b833-9092-40af-bc2e-5800cd1843e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6e2c824b0c9 namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: 91fe278b-bb3e-4340-9991-374a56dbb9b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 05e1f5e2-49a6-4db2-94df-a43dd7e60837 kind: Event lastTimestamp: "2026-07-16T17:57:53Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:53Z" name: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl.18c2d6e32e9c03ec namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: 3c868ebf-af10-4870-b3cf-9fff5d19ffa7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6687f-predictor-769fddcb6f namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: e03d81bf-3926-4882-b689-4c74876ac77b kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: 'Created pod: success-200-isvc-6687f-predictor-769fddcb6f-tfzgl' metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-6687f-predictor-769fddcb6f.18c2d6d26de77f9e namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 93574f7a-09f3-4dae-960a-4b4de37ca77e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 87b1a369-9011-4a7e-8629-64329b834b6f kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: Scaled up replica set success-200-isvc-6687f-predictor-769fddcb6f from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-6687f-predictor.18c2d6d26b7457ce namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: c6411252-207e-47ac-96e1-534f7988c163 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 82068813-69f1-4700-91f5-9a64369802c1 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:11Z" name: success-200-isvc-6687f-predictor.18c2d6d5ea4e9c9b namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 00ae5e4c-ebad-4d02-a816-8a353c5b9706 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 82068813-69f1-4700-91f5-9a64369802c1 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:11Z" name: success-200-isvc-6687f-predictor.18c2d6d5ead8a0e9 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: dfb05ada-63af-400f-86e7-e2e96264a658 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 82068813-69f1-4700-91f5-9a64369802c1 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:41Z" name: success-200-isvc-6687f-predictor.18c2d6dce9146b09 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: ed754a32-a2b6-40db-b6ee-3613782791f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6687f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 82068813-69f1-4700-91f5-9a64369802c1 kind: Event lastTimestamp: "2026-07-16T17:57:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:41Z" name: success-200-isvc-6687f-predictor.18c2d6dce99541f1 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: c597afdb-2a51-42af-ae2f-c1d6e87668ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 83533847-a648-450c-bf44-f18977a434d9 kind: Event lastTimestamp: "2026-07-16T17:57:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:28Z" name: success-200-isvc-6687f.18c2d6d26a8afab1 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 5d5e8add-49a0-4365-bf06-a97912b5a860 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 83533847-a648-450c-bf44-f18977a434d9 kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-6687f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6687f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-6687f.18c2d6d26aeaa04b namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: a383625b-dac8-485b-b0b8-fc9c50bd7345 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 83533847-a648-450c-bf44-f18977a434d9 kind: Event lastTimestamp: "2026-07-16T17:56:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6687f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:41Z" name: success-200-isvc-6687f.18c2d6d26aeaed09 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: d3134faf-cf7f-40a2-8ae1-705449a49c18 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6687f namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: 83533847-a648-450c-bf44-f18977a434d9 kind: Event lastTimestamp: "2026-07-16T17:57:28Z" message: InferenceService [success-200-isvc-6687f] is Ready metadata: creationTimestamp: "2026-07-16T17:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:28Z" name: success-200-isvc-6687f.18c2d6dd6581f55b namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: 2b03a835-3761-49de-8d8a-e34953eb19c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:07:06.766791Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-97269-predictor-b796c5fd5-tzshw to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d763f8b9d4c6 namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: 1f4012f6-9f8e-4d4e-bab3-be5f72608ba1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d764125bc033 namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: 913cdcad-cb9d-4dd2-862a-fffe3899fb4a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d76413c79e91 namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 8b2fc49b-4d59-4d55-a215-43ddf90cf509 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d764172c84a6 namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: 8c477c9b-a7a1-4e52-a228-f62f76e34cfe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d76417bfd7c8 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: 59d442d8-3061-4083-8456-e52f009630f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d76417cd8755 namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: 5953f79c-fb9c-43d0-949c-9c0f6f6c61eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d7641a736891 namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: f0250ff6-d1a7-4d29-961f-281e4175d5e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:07Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d7641aed7e59 namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 54fdd6ea-3751-49a1-a41c-6a287d17d300 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:07:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:44Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d76446c8567c namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 32c99456-285a-4599-b734-31f96e09bfae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:24:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:24:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:24:26Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d855f4831625 namespace: kserve-ci-e2e-test resourceVersion: "33800" uid: 23e94069-f289-453a-a8d2-2f72ccf8b906 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:24:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97269-predictor-b796c5fd5-tzshw namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ef32f086-22fa-4544-a6e3-b6e27db85bfc kind: Event lastTimestamp: "2026-07-16T18:24:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:24:26Z" name: success-200-isvc-97269-predictor-b796c5fd5-tzshw.18c2d855f483bb69 namespace: kserve-ci-e2e-test resourceVersion: "33801" uid: 72df2207-59c5-4ac6-b398-f0f756a1159c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-97269-predictor-b796c5fd5 namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: b6b5e381-2605-4009-8ad8-a99bfd9bdb42 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'Created pod: success-200-isvc-97269-predictor-b796c5fd5-tzshw' metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-97269-predictor-b796c5fd5.18c2d763f862c5bc namespace: kserve-ci-e2e-test resourceVersion: "26080" uid: 5eed2080-2525-48fa-8218-adb13449ecb5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26064" uid: 63807d04-b942-4d0d-acd4-788046dcc2a1 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Scaled up replica set success-200-isvc-97269-predictor-b796c5fd5 from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-97269-predictor.18c2d763f66f7140 namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 6fa003a2-b852-47fb-9c6c-a3f5c4c3aa7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:07:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: f964518a-753c-45ff-a930-fa29316bb64f kind: Event lastTimestamp: "2026-07-16T18:07:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:36Z" name: success-200-isvc-97269-predictor.18c2d7677587ff32 namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 73096dda-98b0-4e13-9670-224a602cf03b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:07:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: f964518a-753c-45ff-a930-fa29316bb64f kind: Event lastTimestamp: "2026-07-16T18:07:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:36Z" name: success-200-isvc-97269-predictor.18c2d76775f5c91b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 61105648-b7e2-4baa-9a8d-f98eadc87ebd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T18:07:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: f964518a-753c-45ff-a930-fa29316bb64f kind: Event lastTimestamp: "2026-07-16T18:08:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:21Z" name: success-200-isvc-97269-predictor.18c2d76e74bab790 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 66104c68-5474-4a13-af02-14fb52186815 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T18:07:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97269-predictor namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: f964518a-753c-45ff-a930-fa29316bb64f kind: Event lastTimestamp: "2026-07-16T18:08:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:21Z" name: success-200-isvc-97269-predictor.18c2d76e753a50e2 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: a9bb61e1-dff5-475c-b2a7-767d2dd6bd20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 958692d4-eaf8-41b2-92b3-1547cfe5914e kind: Event lastTimestamp: "2026-07-16T18:07:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:54Z" name: success-200-isvc-97269.18c2d763f6576feb namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 5c9d210b-cd52-44de-ad00-bb9a6730a7f7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 958692d4-eaf8-41b2-92b3-1547cfe5914e kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-97269": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-97269": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-97269.18c2d763f6bde665 namespace: kserve-ci-e2e-test resourceVersion: "26072" uid: a0aad4a9-d947-42be-a3a4-c3f4f5a51795 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 958692d4-eaf8-41b2-92b3-1547cfe5914e kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-97269": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-97269.18c2d763f6be31bf namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 8c45ea79-78ca-4571-b74b-5020b6379cb2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97269 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 958692d4-eaf8-41b2-92b3-1547cfe5914e kind: Event lastTimestamp: "2026-07-16T18:07:54Z" message: InferenceService [success-200-isvc-97269] is Ready metadata: creationTimestamp: "2026-07-16T18:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:54Z" name: success-200-isvc-97269.18c2d76f04b1cda1 namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: 71958f01-9713-40f9-b787-c1582dae14cd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:06:27.216253Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a019b-predictor-54d5fc55df-ccngm to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:06:27Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75ac354737c namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: b1bb77a6-e726-4fb8-8b37-1bed8d9c6903 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a019b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:27Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75acd3b4319 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 321da074-2cd7-4e8b-a5ae-69e25d4fd98a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b3671be19 namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: a30f3f4c-efc1-41c5-9623-dd4fb3023afb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b380f3e43 namespace: kserve-ci-e2e-test resourceVersion: "25705" uid: 113f414f-569a-4583-a62d-849ecfb40299 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b3ace3d01 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: bd547df1-8943-42c0-b70e-1fd6bc43e31a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b3b403eb5 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 738a73f6-178e-49db-9875-914705160f4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b3b4a00a0 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: f103ae6f-ddc6-41e2-9726-727eca42cbf4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b3ec9fe8a namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: b8388ce7-7933-45ac-a220-c174e67cc85f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75b3f3bdae6 namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: 649da960-ab00-430d-82ad-17a85be2de8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d75ba03d120b namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 4ab4b84a-0fcb-4501-ad24-8cf1ed84d974 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d76b1ed455db namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: a25747d8-ed1f-4d46-8c6d-e8a578d41f8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 02e68e74-6ad3-43ac-bd1d-dabe89c07510 kind: Event lastTimestamp: "2026-07-16T18:07:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:37Z" name: success-200-isvc-a019b-predictor-54d5fc55df-ccngm.18c2d76b1ed4be13 namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 62bf01ff-7a21-47cb-b9c7-a6aae063ddce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a019b-predictor-54d5fc55df namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: ea6ed940-c266-47a1-a4fd-1d69330b21b1 kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: 'Created pod: success-200-isvc-a019b-predictor-54d5fc55df-ccngm' metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:06:27Z" name: success-200-isvc-a019b-predictor-54d5fc55df.18c2d75ac270f0e4 namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 9d311622-6ca1-425d-b8b5-27e4475d23b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25602" uid: 5f84fd4d-2ca0-4462-ba99-2ab9eb890bef kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: Scaled up replica set success-200-isvc-a019b-predictor-54d5fc55df from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:06:27Z" name: success-200-isvc-a019b-predictor.18c2d75abda4e150 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 6a758c99-83a8-4652-a066-317474669efc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T18:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 9bb23121-6b56-49dd-89be-9312f83a72a1 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:27Z" name: success-200-isvc-a019b-predictor.18c2d75e3c522370 namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: b425482b-4226-4ef5-82b2-6af9200956df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T18:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a019b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 9bb23121-6b56-49dd-89be-9312f83a72a1 kind: Event lastTimestamp: "2026-07-16T18:07:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T18:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:07:27Z" name: success-200-isvc-a019b-predictor.18c2d75e3cc7f4f2 namespace: kserve-ci-e2e-test resourceVersion: "26346" uid: fb7c62f4-1a58-4107-a0e8-0a27501a50d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a019b namespace: kserve-ci-e2e-test resourceVersion: "25583" uid: 3d0bf941-b51d-4afd-ac5b-3bf1b7ad2acc kind: Event lastTimestamp: "2026-07-16T18:07:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:17Z" name: success-200-isvc-a019b.18c2d75abdd14f00 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 5935f085-78c2-4716-909a-b32d705a89bf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a019b namespace: kserve-ci-e2e-test resourceVersion: "26256" uid: 3d0bf941-b51d-4afd-ac5b-3bf1b7ad2acc kind: Event lastTimestamp: "2026-07-16T18:07:17Z" message: InferenceService [success-200-isvc-a019b] is Ready metadata: creationTimestamp: "2026-07-16T18:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:07:17Z" name: success-200-isvc-a019b.18c2d7665b847529 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 3bc230c7-e790-4e26-948f-11364312a872 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:51.935271Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a90bb-predictor-675fc99bf9-zwxft to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2ca30b7ad namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 9e1a01b4-b8d3-427e-be56-5505f1371df1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2e39ac6a0 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3aa26b32-342b-4d65-81c9-f3ba241360fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2e5832754 namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 949a1e49-7680-43c6-9298-80dda5cf2e3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2e96ac452 namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: 8185f7da-72db-4c04-8e44-eff398a88717 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2e9e3b464 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: ac5fa398-5fe8-48f7-b607-df71f7f82454 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2e9ef3f99 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 554c0053-861e-42a5-bd44-e44264de7894 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2eca762fb namespace: kserve-ci-e2e-test resourceVersion: "21563" uid: 9d74b609-1f6d-4c6d-921d-212385a836bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:57:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:52Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e2ed1fe1b7 namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 2f48dabb-a5c9-44cd-9481-3e987222ad6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:57:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T17:58:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:29Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d6e3415bfb84 namespace: kserve-ci-e2e-test resourceVersion: "21948" uid: 169a878a-a1ee-451e-8380-3a7a36b3d5c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d763f54f51e7 namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: af641d8c-dd9a-4b4f-ab77-9715c4b6cae1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 77d38b5b-b79d-496b-bbd5-f0baa7173598 kind: Event lastTimestamp: "2026-07-16T18:07:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:07:06Z" name: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft.18c2d763f55005a0 namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: d11e86b2-6487-46b0-b164-131f39f81e28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a90bb-predictor-675fc99bf9 namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: 7dccd168-f939-4f15-8bf9-92c9e841518d kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: 'Created pod: success-200-isvc-a90bb-predictor-675fc99bf9-zwxft' metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-a90bb-predictor-675fc99bf9.18c2d6e2c9d963e7 namespace: kserve-ci-e2e-test resourceVersion: "21510" uid: 01c6b9c5-9408-4bff-82fd-77517c129808 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 021b264d-fa49-482b-9356-623f38c545b6 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: Scaled up replica set success-200-isvc-a90bb-predictor-675fc99bf9 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-a90bb-predictor.18c2d6e2c7ca890a namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: bf98b0a7-93de-4e9d-85bf-0a86a3405031 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 276bf0da-5023-449b-a9a5-a72c6e91c108 kind: Event lastTimestamp: "2026-07-16T17:58:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:36Z" name: success-200-isvc-a90bb-predictor.18c2d6e6464639ce namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: fbb6ee37-29d6-4c26-bd5f-ae97614d3e37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 276bf0da-5023-449b-a9a5-a72c6e91c108 kind: Event lastTimestamp: "2026-07-16T17:58:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:36Z" name: success-200-isvc-a90bb-predictor.18c2d6e646a95509 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: c3d19186-7d33-4583-aca6-3c7a8e2fc15f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: 276bf0da-5023-449b-a9a5-a72c6e91c108 kind: Event lastTimestamp: "2026-07-16T17:59:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:07Z" name: success-200-isvc-a90bb-predictor.18c2d6e9c69cbedd namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: a301e49e-da74-4f17-ad3e-5ff362048504 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a90bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: 276bf0da-5023-449b-a9a5-a72c6e91c108 kind: Event lastTimestamp: "2026-07-16T17:59:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:07Z" name: success-200-isvc-a90bb-predictor.18c2d6e9c76b8130 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: e3404a47-ddfb-4b9c-9617-9f17008a20aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: b014e65e-58b4-4db0-ac71-3c7294e97896 kind: Event lastTimestamp: "2026-07-16T17:58:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:40Z" name: success-200-isvc-a90bb.18c2d6e2c7be3d5a namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: e169e87f-7880-4661-a79e-076afb8988e9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: b014e65e-58b4-4db0-ac71-3c7294e97896 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-a90bb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a90bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-a90bb.18c2d6e2c859e516 namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: 9810d321-6d24-4b49-b948-9844ff704232 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: b014e65e-58b4-4db0-ac71-3c7294e97896 kind: Event lastTimestamp: "2026-07-16T17:57:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a90bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:51Z" name: success-200-isvc-a90bb.18c2d6e2c85a2274 namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: ae1c163d-501f-4452-a0ba-9141081a86fc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a90bb namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: b014e65e-58b4-4db0-ac71-3c7294e97896 kind: Event lastTimestamp: "2026-07-16T17:58:40Z" message: InferenceService [success-200-isvc-a90bb] is Ready metadata: creationTimestamp: "2026-07-16T17:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:40Z" name: success-200-isvc-a90bb.18c2d6edfcc9274f namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: e7e80071-c438-4f29-bf04-2370a23c1711 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:45:16.456073Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e62f1-predictor-7d46c6569f-w87lj to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:45:16Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d632e41efd00 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cf17da4c-a5e8-4ff7-9592-0df6715a8a89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:45:16Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d632fdf66bfc namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: eb3894e3-99ce-4773-aab8-34d2312a3308 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:16Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d632ff7a5807 namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 97d75b5b-e333-44f4-a48c-543dd6ff14dd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" in 15.819s (15.819s including waiting). Image size: 1335611503 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d636ae69711a namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 13f25028-0c83-4944-b2a4-202e3daa0d92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d636b3003569 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 4b57981a-cc25-488d-a5d6-515709fa2b68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d636b37f6ff6 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 76bea603-ee8f-458c-9cca-86238352fcfc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:32Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d636b38c585f namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: 9da07299-8628-40e2-ac36-6454731fc251 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.484s (2.484s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:35Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d63747a3d8ea namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: d3cfcb2c-c4c9-41e1-b5f0-6a1163beb821 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:35Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d6374ab69e50 namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: c0d55a8f-aeb1-460b-adcc-029cb7c2787b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:45:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:35Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d6374b222f61 namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 2dabb302-f05b-4902-97a3-4f609f697335 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:46:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:12Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d63773ebb693 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: f176de24-e7d3-49df-a82b-5344c9e92925 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d647e7e23d34 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 4de1ebd5-dfff-4cb5-a3b5-ff77ae0ce699 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d647e7e2a6d8 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: 189a4edb-3899-4868-bf79-52147dcfa541 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 1161db69-91a5-43f1-b15b-abe28234f18c kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj.18c2d647fc2dd877 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 0283c6b7-e289-4cf7-8351-e88a9d51de88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e62f1-predictor-7d46c6569f namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: 63523679-b463-40b0-81de-f32d24caf84b kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: 'Created pod: success-200-isvc-e62f1-predictor-7d46c6569f-w87lj' metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: success-200-isvc-e62f1-predictor-7d46c6569f.18c2d632e3a5f5d9 namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: a63019b0-2a5c-4789-9784-4b7b1e896a0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 49a358b6-bbd4-47f2-addd-638682f92289 kind: Event lastTimestamp: "2026-07-16T17:45:16Z" message: Scaled up replica set success-200-isvc-e62f1-predictor-7d46c6569f from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:16Z" name: success-200-isvc-e62f1-predictor.18c2d632e04ec773 namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: c3b28c86-e4d7-4811-9e0c-f2f72db0ba94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 0be35ab5-abe2-41c2-bfbc-ea72a035c104 kind: Event lastTimestamp: "2026-07-16T17:45:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:46Z" name: success-200-isvc-e62f1-predictor.18c2d63665fb9143 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: cf54bd9f-8e3b-423c-a910-3930cbbea487 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 0be35ab5-abe2-41c2-bfbc-ea72a035c104 kind: Event lastTimestamp: "2026-07-16T17:45:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:46Z" name: success-200-isvc-e62f1-predictor.18c2d636666dff90 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 76c87bc9-a134-47d5-8f94-f1cfaf40b857 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 0be35ab5-abe2-41c2-bfbc-ea72a035c104 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-e62f1-predictor.18c2d63d65491377 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 63e4690c-07a7-47db-b5a7-165be7fbea15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e62f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 0be35ab5-abe2-41c2-bfbc-ea72a035c104 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-e62f1-predictor.18c2d63d65ce0917 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: 7c14bae9-9f19-4f28-9670-efb1cd31c9a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e62f1 namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 7c1d08c0-20c8-4148-8b7f-d61b994d2dfc kind: Event lastTimestamp: "2026-07-16T17:46:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:22Z" name: success-200-isvc-e62f1.18c2d632e64b8ec5 namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: ecc96c92-86d0-4399-8288-7553d1331fb8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e62f1 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 7c1d08c0-20c8-4148-8b7f-d61b994d2dfc kind: Event lastTimestamp: "2026-07-16T17:46:22Z" message: InferenceService [success-200-isvc-e62f1] is Ready metadata: creationTimestamp: "2026-07-16T17:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:22Z" name: success-200-isvc-e62f1.18c2d6422f2d03c2 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 408a8c92-612b-4f05-bb41-c70b6533a6d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:12.238223Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-efe2d-predictor-564c8f4947-m77mg to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d98c0fd519 namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 038ad919-c407-49e1-a8a4-023e385e859d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9a6137329 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: f0b5daf8-92f5-4488-9752-19edb5d45d53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9a785cef7 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 5d5b8027-1d6f-42b4-9233-cc2b3b78a093 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9aaa83259 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: fbee3977-6ad9-4249-b656-4b4eab4db5db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9ab26c6cb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: eea0fe7b-4b93-4965-b381-41a487fe5116 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9ab2ff4ec namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 1c7ad526-98e3-4d88-9f3e-355d8f19aab2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9ae7dd827 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: a4d98aa9-6321-4ca0-b9e9-82e813c99b52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9af1547cc namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 66df8a33-46aa-479d-badc-7166b9e9f005 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T17:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d6d9e649462c namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: b82cf15e-6287-4026-86ef-b1549e1be1f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:27Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d75abc7de5b1 namespace: kserve-ci-e2e-test resourceVersion: "25601" uid: f0582148-b1b3-4c85-9c38-2d23bf8b8b8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T18:06:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:27Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d75abc7e2549 namespace: kserve-ci-e2e-test resourceVersion: "25604" uid: 31637c51-d6b2-45bc-9e77-f46300b0cf18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 95720232-9c66-4c33-9431-65122d0214fa kind: Event lastTimestamp: "2026-07-16T18:06:29Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:06:29Z" name: success-200-isvc-efe2d-predictor-564c8f4947-m77mg.18c2d75b5ac0bc52 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: e754f52d-4eaf-464e-b1c3-b9ddd16274ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-efe2d-predictor-564c8f4947 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 783a9536-5662-4a3e-988a-58b369b9beb5 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created pod: success-200-isvc-efe2d-predictor-564c8f4947-m77mg' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor-564c8f4947.18c2d6d98ba4f8cb namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: 06bfdc72-5e14-45dd-8e0d-cb82d791209a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: efe41373-14eb-4d20-ac67-2fde93d5071a kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Scaled up replica set success-200-isvc-efe2d-predictor-564c8f4947 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d-predictor.18c2d6d989e5ea2e namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: be6b883d-ffee-4cb4-9132-f9f180d04f5b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:57:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 30cc6cb4-bb90-4666-976f-fcc1aa7bba7b kind: Event lastTimestamp: "2026-07-16T17:58:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:12Z" name: success-200-isvc-efe2d-predictor.18c2d6dd088ffd2d namespace: kserve-ci-e2e-test resourceVersion: "21773" uid: a6841bcc-6126-4d4e-b244-da05b88818b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:57:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 30cc6cb4-bb90-4666-976f-fcc1aa7bba7b kind: Event lastTimestamp: "2026-07-16T17:58:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:12Z" name: success-200-isvc-efe2d-predictor.18c2d6dd09032eca namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: 9ed84ec5-e41c-4cef-bcee-156674c87cac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 30cc6cb4-bb90-4666-976f-fcc1aa7bba7b kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:42Z" name: success-200-isvc-efe2d-predictor.18c2d6e087c8cf93 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 4c886ef5-85b9-4731-8fad-81c3b900c73e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-efe2d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 30cc6cb4-bb90-4666-976f-fcc1aa7bba7b kind: Event lastTimestamp: "2026-07-16T17:57:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:42Z" name: success-200-isvc-efe2d-predictor.18c2d6e0887a67cf namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4cafc846-8e89-48ff-8a37-b6dc5e2cd996 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: b00c87ec-2272-426f-bb8d-1f2d66b685c9 kind: Event lastTimestamp: "2026-07-16T17:57:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:59Z" name: success-200-isvc-efe2d.18c2d6d989f5ca16 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: a6fed1b0-f950-4a80-b244-d5546f510e64 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: b00c87ec-2272-426f-bb8d-1f2d66b685c9 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-efe2d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-efe2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d.18c2d6d98a5cb530 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: c9c08770-e868-40e7-ba5b-d6e30a33af21 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: b00c87ec-2272-426f-bb8d-1f2d66b685c9 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-efe2d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:12Z" name: success-200-isvc-efe2d.18c2d6d98a5cf969 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: bde22a62-2139-48ca-a82a-bbaca8c32a3e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-efe2d namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: b00c87ec-2272-426f-bb8d-1f2d66b685c9 kind: Event lastTimestamp: "2026-07-16T17:57:59Z" message: InferenceService [success-200-isvc-efe2d] is Ready metadata: creationTimestamp: "2026-07-16T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:59Z" name: success-200-isvc-efe2d.18c2d6e4a19b78d3 namespace: kserve-ci-e2e-test resourceVersion: "21653" uid: 6d32497d-79eb-4f4e-bdcd-5375c00cebd0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:46:46.771840Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f0d41-predictor-8bd487845-z6k94 to ip-10-0-139-103.ec2.internal metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d647eb5c3f8e namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 56173f3b-18b8-48e3-a604-6e6f7f7b6c74 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f0d41-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d647f58fd18b namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 659f3ef4-17b7-4fb1-8ecf-b11c8eddc726 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d648284f22f4 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 7f9cd037-8f94-4eb6-b771-52be1424ccf5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d64829bf940f namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 27275340-e741-45ec-8dcd-dfbabf5491da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d6482d30d5e1 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: c2520d93-3a96-4851-a0bf-cb0c7198c60b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d6482da5e101 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: b77bb67b-8dd5-4f2a-87b7-315029abc24a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d6482db673d7 namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: e347ce9b-8e8a-42c5-adb5-0b5c6e90aa6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d648308809ac namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: a70818f4-70c7-4e95-b050-1fe883749a1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:46:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:47Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d64830fa7089 namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 03744202-5bbd-4bdb-9c4b-586fd3c0effe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T17:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:56:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:04Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d64847c9084a namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: b01ebade-aa2e-4596-b184-13a748242995 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:01Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d6c923537168 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 044a6f1a-9e93-40db-8af0-0d2b270979dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:01Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d6c923540133 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 12178729-637a-46ae-8026-70e5c63ad598 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0d41-predictor-8bd487845-z6k94 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 773df83f-1eb5-4d26-a88e-d6bd277ad72e kind: Event lastTimestamp: "2026-07-16T17:56:04Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:04Z" name: success-200-isvc-f0d41-predictor-8bd487845-z6k94.18c2d6c9bc2e11c0 namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 74d37f97-5ffe-4ba5-89f1-0b171df76af9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-103.ec2.internal source: component: kubelet host: ip-10-0-139-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f0d41-predictor-8bd487845 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 89dbb61e-84a3-4ade-a8d0-5f7976f37f73 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'Created pod: success-200-isvc-f0d41-predictor-8bd487845-z6k94' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-f0d41-predictor-8bd487845.18c2d647eaeef624 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: c964aad2-68db-4644-a5b4-cc422349a89a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 04511011-2f36-4c08-9dfd-f9651ae9ea05 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Scaled up replica set success-200-isvc-f0d41-predictor-8bd487845 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-f0d41-predictor.18c2d647e92cbfba namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 0a0edb67-af1d-48b8-a93a-50bb93ffd0e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: b21aa33f-0183-40d1-bfb5-1c0eadf789e7 kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: success-200-isvc-f0d41-predictor.18c2d64b68a1d503 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: ba1a5309-e2e9-4e2b-a96f-222aabfb2f7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: b21aa33f-0183-40d1-bfb5-1c0eadf789e7 kind: Event lastTimestamp: "2026-07-16T17:47:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:16Z" name: success-200-isvc-f0d41-predictor.18c2d64b692668fa namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 8e4a0d56-869c-40e2-97ec-985ba327f6cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: b21aa33f-0183-40d1-bfb5-1c0eadf789e7 kind: Event lastTimestamp: "2026-07-16T17:47:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:46Z" name: success-200-isvc-f0d41-predictor.18c2d65266ecbdf6 namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: 23f97e41-6de9-4ed5-9664-a46e9f573cc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0d41-predictor namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: b21aa33f-0183-40d1-bfb5-1c0eadf789e7 kind: Event lastTimestamp: "2026-07-16T17:47:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:46Z" name: success-200-isvc-f0d41-predictor.18c2d6526764922b namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: 579c45a0-ec0b-4ced-9073-997b9b69fb31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 48675a9b-0a98-4331-851d-489086b724af kind: Event lastTimestamp: "2026-07-16T17:47:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:34Z" name: success-200-isvc-f0d41.18c2d647e8fbc4f5 namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 3b681243-d65d-449d-a7f7-9ff5397800c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 48675a9b-0a98-4331-851d-489086b724af kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'Failed to update status for InferenceService "success-200-isvc-f0d41": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f0d41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-f0d41.18c2d647e96a1bee namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: c0f97a7e-a9b1-4cbc-b262-c95330210c10 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 48675a9b-0a98-4331-851d-489086b724af kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f0d41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:46Z" name: success-200-isvc-f0d41.18c2d647e96a74dd namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 14dd244c-6e7e-4e74-9572-8c9ffcdacb79 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: 48675a9b-0a98-4331-851d-489086b724af kind: Event lastTimestamp: "2026-07-16T17:47:34Z" message: InferenceService [success-200-isvc-f0d41] is Ready metadata: creationTimestamp: "2026-07-16T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:34Z" name: success-200-isvc-f0d41.18c2d65302b69d7b namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 36d20b97-d3d4-4d33-a5e3-fc35efdee19f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T18:08:06.791938Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-97269-f566d894d-nvnf6 to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T18:08:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T18:08:06Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d771f280dba0 namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: 77cff20d-7e3a-4762-a09b-2b519d623297 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:07Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: "2026-07-16T18:08:07Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T18:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T18:08:07Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d7720c8085ef namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: 68188ac3-8b32-435c-b9c2-be71793a7e7d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-97269} kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: "2026-07-16T18:08:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T18:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:07Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d7720dd25e2f namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 464260b0-fdbb-45d7-a28d-ece694d02384 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-97269} kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: "2026-07-16T18:08:07Z" message: 'Created container: switch-graph-97269' metadata: creationTimestamp: "2026-07-16T18:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:07Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d77210c0c73f namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 5f66443b-60ee-4b40-bb22-b44c0fb6f80b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-97269} kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: "2026-07-16T18:08:07Z" message: Started container switch-graph-97269 metadata: creationTimestamp: "2026-07-16T18:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:08:07Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d77211362ccf namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: 0f5b8e58-7bbb-40c7-aeff-8475e3a57818 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:24:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-97269} kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: "2026-07-16T18:24:25Z" message: Stopping container switch-graph-97269 metadata: creationTimestamp: "2026-07-16T18:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:24:25Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d855eb975fc0 namespace: kserve-ci-e2e-test resourceVersion: "33784" uid: 0075583c-a9ee-4431-a884-3bad7fd9aea6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T18:24:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-97269} kind: Pod name: switch-graph-97269-f566d894d-nvnf6 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 9537029c-8eb2-4a36-b628-5b9976805f12 kind: Event lastTimestamp: "2026-07-16T18:24:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T18:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:24:53Z" name: switch-graph-97269-f566d894d-nvnf6.18c2d8569b7155b0 namespace: kserve-ci-e2e-test resourceVersion: "34061" uid: f2c15c2b-1851-4f7f-ab6f-9366769a5dfa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-97269-f566d894d namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 161f65d2-bfd4-4130-8d47-8adb3c492e6c kind: Event lastTimestamp: "2026-07-16T18:08:06Z" message: 'Created pod: switch-graph-97269-f566d894d-nvnf6' metadata: creationTimestamp: "2026-07-16T18:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:06Z" name: switch-graph-97269-f566d894d.18c2d771f21e7314 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: b49c0e34-b9f5-4e11-94f9-37b8ddf18630 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-97269 namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: 62530480-e2d3-41ba-9427-af19f909b031 kind: Event lastTimestamp: "2026-07-16T18:08:06Z" message: Scaled up replica set switch-graph-97269-f566d894d from 0 to 1 metadata: creationTimestamp: "2026-07-16T18:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:06Z" name: switch-graph-97269.18c2d771f1581d0e namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 4fe675a1-cba2-4d0f-858d-2ad5820bea6e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-97269 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 9e2f33af-7862-446b-9157-5573746d05ac kind: Event lastTimestamp: "2026-07-16T18:08:06Z" message: InferenceGraph [switch-graph-97269] is Ready metadata: creationTimestamp: "2026-07-16T18:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:08:06Z" name: switch-graph-97269.18c2d771f6c00e3b namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: b30273eb-e742-48aa-bcaa-9c510d363d96 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-97269 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 7f2cd4ff-039d-4078-8f60-e7ed98bab1c3 kind: Event lastTimestamp: "2026-07-16T18:08:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:36Z" name: switch-graph-97269.18c2d77570b9a240 namespace: kserve-ci-e2e-test resourceVersion: "27151" uid: cae47b90-204f-4d66-b6ba-85843cba924e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T18:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-97269 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 7f2cd4ff-039d-4078-8f60-e7ed98bab1c3 kind: Event lastTimestamp: "2026-07-16T18:08:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T18:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:08:36Z" name: switch-graph-97269.18c2d775710d5457 namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 2cf5ca7a-3cc2-4af0-98ea-333f722a37ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:46:36.491958Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e62f1-6549dd5fcb-8hnkl to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:46:36Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d64586a1a9f0 namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: d8ab23a1-3318-4516-8253-2e067a667683 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:36Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:46:36Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:46:36Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d645a00eacf8 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 86602ce8-df69-4af0-91d5-f37309df7192 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e62f1} kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:46:36Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:36Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d645a14da0b5 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 6d38d735-5db6-4879-b2ea-17f320913438 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e62f1} kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:46:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" in 2.215s (2.215s including waiting). Image size: 218081401 bytes.' metadata: creationTimestamp: "2026-07-16T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:39Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d64625547f61 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 83c540dd-f3fa-475b-896f-9834e0150485 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e62f1} kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:46:39Z" message: 'Created container: switch-graph-e62f1' metadata: creationTimestamp: "2026-07-16T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:39Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d646285df6f3 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: cd96bf0e-6b3f-4b1d-acc1-55eaf2292bee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e62f1} kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:46:39Z" message: Started container switch-graph-e62f1 metadata: creationTimestamp: "2026-07-16T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:39Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d64628c85118 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 25b084f7-2902-41e2-9b0e-813f47edaf2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e62f1} kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: Stopping container switch-graph-e62f1 metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d647e0880739 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 43e8c3b6-bf62-4c78-a04c-a0f7bfd1a543 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e62f1} kind: Pod name: switch-graph-e62f1-6549dd5fcb-8hnkl namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 83ce8ffe-1268-4803-b97e-dab63f0a623f kind: Event lastTimestamp: "2026-07-16T17:47:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:15Z" name: switch-graph-e62f1-6549dd5fcb-8hnkl.18c2d648e16ba8f3 namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 61491f56-2571-4bb1-b587-8d9304f46f80 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e62f1-6549dd5fcb namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: c7a2f114-5daa-4734-9ed1-57ba9f9d731d kind: Event lastTimestamp: "2026-07-16T17:46:36Z" message: 'Created pod: switch-graph-e62f1-6549dd5fcb-8hnkl' metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:36Z" name: switch-graph-e62f1-6549dd5fcb.18c2d645861ce31f namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 16583bb9-e0b5-4de0-a317-7a61adc6cf25 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e62f1 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 9ea6af41-b772-4a6a-ba98-e43fd7a0456e kind: Event lastTimestamp: "2026-07-16T17:46:36Z" message: Scaled up replica set switch-graph-e62f1-6549dd5fcb from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:36Z" name: switch-graph-e62f1.18c2d64584f3e772 namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 11df9b9b-2971-4b66-9952-8a8c25a173e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e62f1 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: dcddc5de-7c2b-4d78-a34e-0694421a5e99 kind: Event lastTimestamp: "2026-07-16T17:46:36Z" message: InferenceGraph [switch-graph-e62f1] is Ready metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:36Z" name: switch-graph-e62f1.18c2d6458b5e62cb namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: ac976dc6-15ee-4fc6-a66e-3f9ca101f2c2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:47:46.833758Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f0d41-64565558d8-5ldxn to ip-10-0-135-249.ec2.internal metadata: creationTimestamp: "2026-07-16T17:47:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:47:46Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d655e7545cc9 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 6405b4bd-fabc-4cce-97ea-cd2946a4a3bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65b4bb664b-hmz5q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: "2026-07-16T17:47:47Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:47:47Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d656016e88ac namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 2e7cf718-daa2-442d-be3e-12c8ca612f6b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0d41} kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: "2026-07-16T17:47:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:47Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d65602cc959c namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: da9c6c22-6c4e-4144-b8a3-90a215c8b59e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0d41} kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: "2026-07-16T17:47:47Z" message: 'Created container: switch-graph-f0d41' metadata: creationTimestamp: "2026-07-16T17:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:47Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d65605ef63d4 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: ae18df92-c20b-408e-bb7a-1788f3829925 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0d41} kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: "2026-07-16T17:47:47Z" message: Started container switch-graph-f0d41 metadata: creationTimestamp: "2026-07-16T17:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:47:47Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d656066ace09 namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 91308e6a-3dd9-4cc2-9b0e-7d43804b03d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0d41} kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: "2026-07-16T17:56:01Z" message: Stopping container switch-graph-f0d41 metadata: creationTimestamp: "2026-07-16T17:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:01Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d6c919b9cd0b namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: d591de95-7c08-4b88-96a1-14b7dc9f4df0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0d41} kind: Pod name: switch-graph-f0d41-64565558d8-5ldxn namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 3fe02a67-a134-4110-a5c4-da76d3fcbabe kind: Event lastTimestamp: "2026-07-16T17:56:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:29Z" name: switch-graph-f0d41-64565558d8-5ldxn.18c2d6c9b6239c0f namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: db4337d1-e428-4c9b-a3b3-5cc6cf468782 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-249.ec2.internal source: component: kubelet host: ip-10-0-135-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f0d41-64565558d8 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 1153096c-d708-47f8-bc70-cad48868807f kind: Event lastTimestamp: "2026-07-16T17:47:46Z" message: 'Created pod: switch-graph-f0d41-64565558d8-5ldxn' metadata: creationTimestamp: "2026-07-16T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:46Z" name: switch-graph-f0d41-64565558d8.18c2d655e6f672eb namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 7e01bc53-d959-42de-aa6c-bacddebf9f27 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: ca3d3026-b5ba-4d21-81e0-09e90144a950 kind: Event lastTimestamp: "2026-07-16T17:47:46Z" message: Scaled up replica set switch-graph-f0d41-64565558d8 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:46Z" name: switch-graph-f0d41.18c2d655e609e3e4 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 82aa8818-efa6-4c67-8d96-ffd419655645 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: bdac3a7c-ff1a-4191-a027-9f3eb77b9657 kind: Event lastTimestamp: "2026-07-16T17:47:46Z" message: InferenceGraph [switch-graph-f0d41] is Ready metadata: creationTimestamp: "2026-07-16T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:46Z" name: switch-graph-f0d41.18c2d655eb04dc8c namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: e5f69f49-e678-45dd-9b40-00c884fda192 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: ca786406-e9f6-4baa-a674-14a3ff74ed09 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:16Z" name: switch-graph-f0d41.18c2d659649e989b namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 1bb97b84-3c26-4492-88bb-83109c4e0274 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f0d41 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: ca786406-e9f6-4baa-a674-14a3ff74ed09 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:16Z" name: switch-graph-f0d41.18c2d65964e2e559 namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: 7472d6e0-a7c8-42e9-b1fc-3ad8e7a123f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34070"